Skip to content

Automate browser based workflows with AI

Revert the six flag-gated agent-handler verification PRs for SKY-12398/12399/12400 (#7590)

Shuchang Zheng584eb7896599
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index a00e5ffe8..1f98176a4 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -207,38 +207,6 @@ class CardNumberInputMismatch(SkyvernException):
)
-class MaskedInputReadbackMismatch(SkyvernException):
- # A masked field (postal code, SSN, etc.) accepted the keystrokes but committed nothing usable —
- # it reads back empty or still shows mask placeholders. Counts only; the raw value may be a secret.
- def __init__(self, *, expected_char_count: int, committed_char_count: int):
- self.expected_char_count = expected_char_count
- self.committed_char_count = committed_char_count
- super().__init__(
- "Masked input read-back mismatch: "
- f"expected {expected_char_count} characters, committed {committed_char_count}."
- )
-
-
-class InputTextCommitMismatch(SkyvernException):
- def __init__(self, *, expected_length: int, actual_length: int):
- self.expected_length = expected_length
- self.actual_length = actual_length
- super().__init__(
- f"Input text commit mismatch: expected length {expected_length}, found actual length {actual_length}."
- )
-
-
-class RepeatedActionFailure(SkyvernException):
- def __init__(self, *, element_id: str, attempt_count: int, failure_description: str):
- self.element_id = element_id
- self.attempt_count = attempt_count
- self.failure_description = failure_description
- super().__init__(
- f"Action on element {element_id} failed identically {attempt_count} consecutive times: "
- f"{failure_description}."
- )
-
-
class SecretInputMismatch(SkyvernException):
def __init__(self) -> None:
# No secret material in the message: not the value, its length, or its character classes.
@@ -1071,17 +1039,9 @@ class NoElementBoudingBox(SkyvernException):
super().__init__(f"Element does not have a bounding box. element_id={element_id}")
-class AutoCompletionCommitFailure(SkyvernException):
- def __init__(self, stage: str, attempt_trail: list[str] | None = None) -> None:
- trail = attempt_trail or [stage]
- super().__init__(f"Autocomplete selection failed. stage={stage} attempt_trail={','.join(trail)}")
- self.stage = stage
- self.attempt_trail = trail
-
-
-# Deprecated: superseded by AutoCompletionCommitFailure, which carries the failing stage.
-# Kept for one release so downstream `except` clauses keep working.
-NoIncrementalElementFoundForAutoCompletion = AutoCompletionCommitFailure
+class NoIncrementalElementFoundForAutoCompletion(SkyvernException):
+ def __init__(self, element_id: str, text: str) -> None:
+ super().__init__(f"No auto completion shown up after fill in [{text}]. element_id={element_id}")
class NoSuitableAutoCompleteOption(SkyvernException):
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 60fc1dd7e..8520f2683 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -57,7 +57,6 @@ from skyvern.exceptions import (
MissingExtractActionsResponse,
NoTOTPVerificationCodeFound,
PDFEmbedBase64DecodeError,
- RepeatedActionFailure,
ScrapingFailed,
SkyvernException,
StepTerminationError,
@@ -169,7 +168,7 @@ from skyvern.webeye.actions.parse_actions import (
parse_cua_actions,
parse_ui_tars_actions,
)
-from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
+from skyvern.webeye.actions.responses import ActionResult, ActionSuccess
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.cdp_download_interceptor import download_filename_from_suffix
from skyvern.webeye.scraper.scraped_page import ElementTreeFormat, ScrapedPage
@@ -181,103 +180,6 @@ RECOVERABLE_BLANK_WORKFLOW_TASK_URLS = {":"}
EXTRACT_ACTION_TEMPLATE = "extract-action"
DECISIVE_CRITERION_VALIDATE_TEMPLATE = "decisive-criterion-validate"
-REPEATED_ACTION_FAILURE_LIMIT = 3
-REPEATED_ACTION_CIRCUIT_BREAKER_FLAG = "REPEATED_ACTION_CIRCUIT_BREAKER"
-_ACTION_REPEAT_PAYLOAD_FIELDS = (
- "button",
- "download",
- "duration",
- "file_name",
- "file_url",
- "hold",
- "hold_seconds",
- "is_checked",
- "is_magic_link",
- "keys",
- "option",
- "path",
- "repeat",
- "start_x",
- "start_y",
- "text",
- "url",
- "verification_code",
- "x",
- "y",
-)
-_PERSISTED_ACTION_REPEAT_PAYLOAD_FIELDS = tuple(
- field for field in _ACTION_REPEAT_PAYLOAD_FIELDS if field in Action.model_fields
-)
-
-
-def _repeated_action_identity(action: Action) -> tuple[ActionType, str, str] | None:
- if not action.action_type.is_web_action() or not action.element_id:
- return None
- payload = {field: getattr(action, field) for field in _PERSISTED_ACTION_REPEAT_PAYLOAD_FIELDS}
- payload_digest = hashlib.sha256(
- json.dumps(payload, default=str, sort_keys=True, separators=(",", ":")).encode()
- ).hexdigest()
- return action.action_type, action.element_id, payload_digest
-
-
-def _get_repeated_action_failure(
- attempts: list[tuple[Action, list[ActionResult]]],
-) -> RepeatedActionFailure | None:
- recent_attempts = attempts[-REPEATED_ACTION_FAILURE_LIMIT:]
- if len(recent_attempts) < REPEATED_ACTION_FAILURE_LIMIT:
- return None
- identities = [_repeated_action_identity(action) for action, _ in recent_attempts]
- if identities[0] is None or any(identity != identities[0] for identity in identities[1:]):
- return None
- if any(not results or any(result.success for result in results) for _, results in recent_attempts):
- return None
- _, element_id, _ = identities[0]
- final_result = recent_attempts[-1][1][-1]
- failure_description = final_result.exception_type or "UnknownFailure"
- return RepeatedActionFailure(
- element_id=element_id,
- attempt_count=REPEATED_ACTION_FAILURE_LIMIT,
- failure_description=failure_description,
- )
-
-
-async def _is_repeated_action_breaker_enabled(task: Task) -> bool:
- try:
- distinct_id = task.workflow_run_id if task.workflow_run_id else task.task_id
- return bool(
- await app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached(
- REPEATED_ACTION_CIRCUIT_BREAKER_FLAG,
- distinct_id,
- properties={"organization_id": task.organization_id},
- )
- )
- except Exception:
- LOG.warning(
- "Failed to evaluate repeated-action circuit breaker flag; defaulting to disabled",
- task_id=task.task_id,
- exc_info=True,
- )
- return False
-
-
-async def _get_previous_action_attempts(task: Task, current_step_id: str) -> list[tuple[Action, list[ActionResult]]]:
- try:
- steps = await app.DATABASE.tasks.get_task_steps(
- task_id=task.task_id,
- organization_id=task.organization_id,
- )
- except Exception:
- LOG.warning("Failed to load action history for repeated-action detection", task_id=task.task_id, exc_info=True)
- return []
- if not isinstance(steps, list):
- return []
- attempts: list[tuple[Action, list[ActionResult]]] = []
- for previous_step in sorted(steps, key=lambda item: (item.order, item.retry_index)):
- if previous_step.step_id == current_step_id:
- continue
- if previous_step.output and previous_step.output.actions_and_results:
- attempts.extend(previous_step.output.actions_and_results)
- return attempts
async def resolve_inherited_workflow_task_page(browser_state: BrowserState, workflow_run_id: str) -> Page:
@@ -1875,7 +1777,6 @@ class ForgeAgent:
element_id_to_action_index[action.element_id] = action_idx
element_id_to_last_action: dict[str, int] = dict()
- previous_step_attempts: list[tuple[Action, list[ActionResult]]] | None = None
try:
wait_config = await get_or_create_wait_config(task.task_id, task.workflow_run_id, task.organization_id)
base_delay = get_wait_time(wait_config, "inter_action_delay", default=0.5)
@@ -1990,42 +1891,6 @@ class ForgeAgent:
action,
results,
)
- if (
- results
- and not any(result.success for result in results)
- and await _is_repeated_action_breaker_enabled(task)
- ):
- current_step_attempts = detailed_agent_step_output.actions_and_results[: action_idx + 1]
- repeated_failure = _get_repeated_action_failure(current_step_attempts)
- current_identity = _repeated_action_identity(action)
- current_attempts_can_follow_history = current_identity is not None and all(
- _repeated_action_identity(attempt_action) == current_identity
- and attempt_results
- and not any(result.success for result in attempt_results)
- for attempt_action, attempt_results in current_step_attempts
- )
- if (
- repeated_failure is None
- and len(current_step_attempts) < REPEATED_ACTION_FAILURE_LIMIT
- and current_attempts_can_follow_history
- ):
- if previous_step_attempts is None:
- previous_step_attempts = await _get_previous_action_attempts(task, step.step_id)
- repeated_failure = _get_repeated_action_failure(previous_step_attempts + current_step_attempts)
- if repeated_failure is not None:
- final_result = results[-1]
- repeated_result = ActionFailure(
- repeated_failure,
- stop_execution_on_failure=final_result.stop_execution_on_failure,
- download_triggered=final_result.download_triggered,
- interacted_with_sibling=bool(final_result.interacted_with_sibling),
- interacted_with_parent=bool(final_result.interacted_with_parent),
- )
- repeated_result.skip_remaining_actions = final_result.skip_remaining_actions
- repeated_result.needs_followup = final_result.needs_followup
- repeated_result.followup_message = final_result.followup_message
- results = [repeated_result]
- detailed_agent_step_output.actions_and_results[action_idx] = (action, results)
# Determine wait time between actions
wait_time = random.uniform(base_delay, base_delay * 2) if base_delay > 0 else 0.0
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index 999d8b2ce..2f90646c4 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -33,7 +33,6 @@ from skyvern.constants import (
from skyvern.core.script_generations.fuzzy_matcher import match_option_exact_or_stem
from skyvern.errors.errors import TOTPExpiredError, UserDefinedError, filter_to_user_defined_codes
from skyvern.exceptions import (
- AutoCompletionCommitFailure,
CaptchaSolveError,
CardNumberInputMismatch,
EmptySelect,
@@ -48,13 +47,11 @@ from skyvern.exceptions import (
IllegitComplete,
ImaginaryFileUrl,
ImaginarySecretValue,
- InputTextCommitMismatch,
InputToInvisibleElement,
InputToReadonlyElement,
InteractWithDisabledElement,
InteractWithDropdownContainer,
InvalidElementForTextInput,
- MaskedInputReadbackMismatch,
MissingElement,
MissingElementDict,
MissingElementInCSSMap,
@@ -63,6 +60,7 @@ from skyvern.exceptions import (
NoAutoCompleteOptionMeetCondition,
NoAvailableOptionFoundForCustomSelection,
NoElementMatchedForTargetOption,
+ NoIncrementalElementFoundForAutoCompletion,
NoIncrementalElementFoundForCustomSelection,
NoSuitableAutoCompleteOption,
NoTOTPSecretFound,
@@ -177,10 +175,6 @@ LOG = structlog.get_logger()
UPLOAD_PENDING_FOLLOWUP_MESSAGE = "Upload is not complete yet. Continue the upload flow."
FIX_TEL_INPUT_DIGIT_DROP_FLAG = "FIX_TEL_INPUT_DIGIT_DROP"
-VERIFY_EMERGING_SELECT_PICK_FLAG = "VERIFY_EMERGING_SELECT_PICK"
-FIX_MASKED_INPUT_READBACK_FLAG = "FIX_MASKED_INPUT_READBACK"
-SURFACE_BLOCKED_FORM_ADVANCE_FLAG = "SURFACE_BLOCKED_FORM_ADVANCE"
-VERIFY_INPUT_TEXT_COMMIT_FLAG = "VERIFY_INPUT_TEXT_COMMIT"
COLLAPSE_SELECT_FANOUT_FLAG = "COLLAPSE_SELECT_FANOUT"
COLLAPSE_CUSTOM_SELECT_FANOUT_FLAG = "COLLAPSE_CUSTOM_SELECT_FANOUT"
COLLAPSE_AUTOCOMPLETE_FANOUT_FLAG = "COLLAPSE_AUTOCOMPLETE_FANOUT"
@@ -218,31 +212,6 @@ DOWNLOAD_EVENT_ACTIVE_DIR_GRACE_SECONDS = 60
DOWNLOAD_DUPLICATE_STEM_SUFFIX_RE = re.compile(r"(?:\s+\(\d{1,3}\)|_\d{1,3})$")
SELECT_SHADOW_MATCH_APOSTROPHE_RE = re.compile(r"['`‘’]")
SELECT_SHADOW_MATCH_WORD_RE = re.compile(r"\w+")
-AUTOCOMPLETE_COMMIT_REQUIRED_FLAG = "AUTOCOMPLETE_COMMIT_REQUIRED"
-AUTOCOMPLETE_COMMIT_WORD_RE = re.compile(r"[^\W_]+")
-AUTOCOMPLETE_POLL_INTERVAL_SECONDS = 0.5
-AUTOCOMPLETE_POLL_MAX_ATTEMPTS = 16
-AUTOCOMPLETE_POLL_DEFAULT_BUDGET_SECONDS = 4.0
-AUTOCOMPLETE_TOTAL_DEADLINE_SECONDS = 8.0
-AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS = 2.0
-AUTOCOMPLETE_TEARDOWN_ERROR_MARKERS = (
- "context was destroyed",
- "navigation",
- "target closed",
- "has been closed",
- "frame was detached",
-)
-AUTOCOMPLETE_DETACHED_ERROR_MARKERS = (
- "not attached",
- "detached",
-)
-AUTOCOMPLETE_RERESOLVE_FAILURE_MARKERS = (
- "timeout",
- "timed out",
- "not found",
- "no element",
- "failed to find",
-)
def _select_shadow_match_enabled() -> bool:
@@ -469,16 +438,6 @@ def _resolve_autocomplete_candidate(
return matched_index, candidates[matched_index]
-def _autocomplete_candidate_for_element_id(
- elements: list[dict], element_id: str
-) -> tuple[int, dict[str, str | None]] | None:
- candidates = _autocomplete_candidates_from_elements(elements)
- for index, candidate in enumerate(candidates):
- if candidate.get("element_id") == element_id:
- return index, candidate
- return None
-
-
async def _read_autocomplete_option_identity(
*,
skyvern_frame: SkyvernFrame,
@@ -515,202 +474,39 @@ async def _verify_autocomplete_option_identity(
LOG.info(
"Autocomplete option index differed from deterministic candidate; accepting label match",
expected_index=matched_index,
- expected_label_length=len(matched_label),
+ expected_label=matched_label,
actual_index=actual_index,
- actual_label_length=len(str(actual_label or "")),
+ actual_label=actual_label,
)
return True
LOG.info(
"Autocomplete option identity did not match deterministic candidate",
expected_index=matched_index,
- expected_label_length=len(matched_label),
+ expected_label=matched_label,
actual_index=actual_index,
- actual_label_length=len(str(actual_label or "")),
+ actual_label=actual_label,
)
return False
-class AutocompleteCommitOutcome(BaseModel):
- committed: bool = False
- side_effects_observed: bool = False
-
-
-def _autocomplete_commit_tokens(value: str | None) -> list[str]:
- return AUTOCOMPLETE_COMMIT_WORD_RE.findall(str(value or "").casefold())
-
-
-async def _read_autocomplete_committed_value(skyvern_element: SkyvernElement) -> tuple[str | None, bool]:
- detached_error_observed = False
- for attempt in range(2):
- try:
- async with asyncio.timeout(AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS):
- value = await get_input_value(skyvern_element.get_tag_name(), skyvern_element.get_locator())
- return str(value or ""), False
- except Exception as exc:
- message = str(exc).casefold()
- if any(marker in message for marker in AUTOCOMPLETE_TEARDOWN_ERROR_MARKERS):
- return None, True
- detached_error = any(marker in message for marker in AUTOCOMPLETE_DETACHED_ERROR_MARKERS)
- if attempt == 0 and detached_error:
- detached_error_observed = True
- continue
- reresolution_failed = (
- detached_error
- or isinstance(exc, TimeoutError)
- or any(marker in message for marker in AUTOCOMPLETE_RERESOLVE_FAILURE_MARKERS)
- )
- if detached_error_observed and reresolution_failed:
- LOG.info(
- "Autocomplete read-back element stayed unreachable after detach, treating as implicit commit",
- element_id=skyvern_element.get_id(),
- error=type(exc).__name__,
- )
- return None, True
- LOG.info(
- "Autocomplete read-back could not read the input value",
- element_id=skyvern_element.get_id(),
- error=type(exc).__name__,
- )
- return None, False
- return None, False
-
-
-async def _autocomplete_suggestions_closed(locator: Locator | None) -> bool | None:
- if locator is None:
- return None
- try:
- async with asyncio.timeout(AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS):
- if await locator.count() == 0:
- return True
- return not await locator.is_visible()
- except Exception:
- return None
-
-
-async def _autocomplete_selection_state(locator: Locator | None) -> bool | None:
- if locator is None:
- return None
- try:
- async with asyncio.timeout(AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS):
- aria_selected = await locator.get_attribute("aria-selected")
- except Exception:
- return None
- if aria_selected is None:
- return None
- return str(aria_selected).casefold() == "true"
-
-
-async def _autocomplete_input_reports_closed(skyvern_element: SkyvernElement) -> bool | None:
- try:
- async with asyncio.timeout(AUTOCOMPLETE_READBACK_TIMEOUT_SECONDS):
- # Must be a live read: get_attr defaults to static-first, and an input scraped before the
- # dropdown opened carries a cached aria-expanded="false" that would read as commit evidence
- # no matter what the keypress actually did.
- aria_expanded = await skyvern_element.get_attr("aria-expanded", mode="dynamic")
- except Exception:
- return None
- if aria_expanded is None:
- return None
- normalized = str(aria_expanded).casefold()
- if normalized not in ("true", "false"):
- return None
- return normalized == "false"
-
-
async def _verify_autocomplete_input_readback(
*,
skyvern_element: SkyvernElement,
+ matched_index: int,
matched_label: str,
- typed_prefix: str,
- pre_action_value: str | None = None,
- suggestions_closed: bool | None = None,
- selection_state_present: bool | None = None,
- allow_unchanged_value: bool = False,
- commit_required: bool = False,
-) -> AutocompleteCommitOutcome:
- actual_value, teardown_observed = await _read_autocomplete_committed_value(skyvern_element)
- if teardown_observed:
- return AutocompleteCommitOutcome(committed=True, side_effects_observed=True)
- if actual_value is None:
- return AutocompleteCommitOutcome(committed=False, side_effects_observed=bool(suggestions_closed))
-
- actual_tokens = _autocomplete_commit_tokens(actual_value)
- matched_tokens = _autocomplete_commit_tokens(matched_label)
- baseline = typed_prefix if pre_action_value is None else pre_action_value
- baseline_tokens = _autocomplete_commit_tokens(baseline)
- actual_normalized = " ".join(actual_tokens)
- matched_normalized = " ".join(matched_tokens)
- baseline_normalized = " ".join(baseline_tokens)
- prefix_normalized = " ".join(_autocomplete_commit_tokens(typed_prefix))
-
- value_changed = actual_normalized != baseline_normalized
- side_effects_observed = value_changed or bool(suggestions_closed)
- actual_set = set(actual_tokens)
- matched_set = set(matched_tokens)
- # Field shows the whole label (plus any prefix like "Selected:"), or a shorter read-back that still
- # covers at least half the label - a lone token of a multi-word label does not identify the selection.
- token_superset = matched_set <= actual_set
- token_subset = actual_set <= matched_set and len(actual_set) * 2 >= len(matched_set)
- token_match = bool(actual_tokens) and bool(matched_tokens) and (token_superset or token_subset)
-
- committed = False
- if allow_unchanged_value:
- committed = token_match or (bool(actual_tokens) and actual_normalized == prefix_normalized)
- elif not actual_tokens:
- # Chip/token widgets clear the input after committing, so cleared + closed listbox is a commit.
- # Under commit-required a dismissal click that clears the field looks identical, so an
- # affirmative selection signal is needed on top of the closed listbox.
- committed = bool(suggestions_closed) and (not commit_required or bool(selection_state_present))
- elif token_match:
- label_distinguishable = matched_normalized != baseline_normalized
- if label_distinguishable:
- committed = value_changed
- else:
- committed = value_changed or bool(suggestions_closed) or bool(selection_state_present)
-
- if not committed:
- LOG.info(
- "Autocomplete read-back did not confirm selected suggestion",
- element_id=skyvern_element.get_id(),
- expected_length=len(matched_normalized),
- actual_length=len(actual_normalized),
- value_changed=value_changed,
- suggestions_closed=suggestions_closed,
- )
- return AutocompleteCommitOutcome(committed=committed, side_effects_observed=side_effects_observed)
-
+) -> bool:
+ actual_value = await get_input_value(skyvern_element.get_tag_name(), skyvern_element.get_locator())
+ if _normalize_select_shadow_text(actual_value) == _normalize_select_shadow_text(matched_label):
+ return True
-async def _poll_autocomplete_incremental_elements(
- *,
- incremental_scraped: IncrementalScrapePage,
- cleanup_factory: CleanupElementTreeFunc,
- deadline: float | None = None,
-) -> list[dict]:
- loop = asyncio.get_running_loop()
- if deadline is None:
- deadline = loop.time() + AUTOCOMPLETE_POLL_DEFAULT_BUDGET_SECONDS
- extracted_elements: list[dict] = []
- for _ in range(AUTOCOMPLETE_POLL_MAX_ATTEMPTS):
- try:
- async with asyncio.timeout(AUTOCOMPLETE_POLL_INTERVAL_SECONDS):
- probe_count = int(await incremental_scraped.get_incremental_elements_num())
- except Exception:
- probe_count = 0
- # Re-extract on every positive probe rather than only on a count change: a listbox can swap a
- # loading placeholder for an option in place, leaving the incremental count constant. We return
- # as soon as an option candidate renders, so the common case still extracts at most once.
- if probe_count > 0:
- incremental_elements = await incremental_scraped.get_incremental_element_tree(cleanup_factory)
- if incremental_elements:
- extracted_elements = incremental_elements
- if _autocomplete_candidates_from_elements(incremental_elements):
- return extracted_elements
- remaining = deadline - loop.time()
- if remaining <= 0:
- break
- await asyncio.sleep(min(AUTOCOMPLETE_POLL_INTERVAL_SECONDS, remaining))
- return extracted_elements
+ LOG.info(
+ "Autocomplete read-back did not match deterministic option",
+ expected_index=matched_index,
+ expected_label=matched_label,
+ actual_value=actual_value,
+ )
+ return False
async def _reset_autocomplete_for_llm_fallback(
@@ -724,7 +520,6 @@ async def _reset_autocomplete_for_llm_fallback(
text: str,
task: Task,
step: Step,
- poll_deadline: float | None = None,
) -> tuple[IncrementalScrapePage, list[dict], list[dict], str, list[str]]:
await current_incremental_scraped.stop_listen_dom_increment()
await skyvern_element.input_clear()
@@ -732,14 +527,13 @@ async def _reset_autocomplete_for_llm_fallback(
incremental_scraped = IncrementalScrapePage(skyvern_frame=skyvern_frame)
await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
await skyvern_element.press_fill(text)
- incremental_element = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scraped,
- cleanup_factory=clean_and_remove_element_tree_factory(
+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1, caller="autocomplete.fallback_refill")
+ incremental_element = await incremental_scraped.get_incremental_element_tree(
+ clean_and_remove_element_tree_factory(
task=task,
step=step,
check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)],
),
- deadline=poll_deadline,
)
if len(incremental_element) > 0:
@@ -757,7 +551,7 @@ async def _reset_autocomplete_for_llm_fallback(
if (await dom_after_open.get_skyvern_element_by_id(element_id)).is_interactable()
]
if len(new_interactable_element_ids) == 0:
- raise AutoCompletionCommitFailure(stage="suggestions_never_rendered")
+ raise NoIncrementalElementFoundForAutoCompletion(element_id=skyvern_element.get_id(), text=text)
LOG.info(
"New elements detected after resetting autocomplete fallback input",
@@ -1388,7 +1182,7 @@ def _plan_tel_text(*, is_tel: bool, is_secret: bool, value: str, pattern: str |
return value, False, is_tel and not is_secret
-async def _is_org_flag_enabled(task: Task, flag: str, log_label: str) -> bool:
+async def _is_tel_digit_fix_enabled(task: Task) -> bool:
organization_id = task.organization_id
if not organization_id:
return False
@@ -1399,44 +1193,20 @@ async def _is_org_flag_enabled(task: Task, flag: str, log_label: str) -> bool:
# Bucket by org (not per-run) for a stable, monitorable ramp and clean rollback.
return bool(
await experimentation_provider.is_feature_enabled_cached(
- flag,
+ FIX_TEL_INPUT_DIGIT_DROP_FLAG,
organization_id,
properties={"organization_id": organization_id},
)
)
except Exception:
LOG.warning(
- f"Failed to evaluate {log_label} flag; defaulting to disabled",
+ "Failed to evaluate tel-digit-fix flag; defaulting to disabled",
organization_id=organization_id,
exc_info=True,
)
return False
-async def _is_tel_digit_fix_enabled(task: Task) -> bool:
- return await _is_org_flag_enabled(task, FIX_TEL_INPUT_DIGIT_DROP_FLAG, "tel-digit-fix")
-
-
-async def _is_verify_emerging_select_pick_enabled(task: Task) -> bool:
- return await _is_org_flag_enabled(task, VERIFY_EMERGING_SELECT_PICK_FLAG, "verify-emerging-select-pick")
-
-
-async def _is_masked_input_readback_fix_enabled(task: Task) -> bool:
- return await _is_org_flag_enabled(task, FIX_MASKED_INPUT_READBACK_FLAG, "masked-input-readback")
-
-
-async def _is_surface_blocked_form_advance_enabled(task: Task) -> bool:
- return await _is_org_flag_enabled(task, SURFACE_BLOCKED_FORM_ADVANCE_FLAG, "surface-blocked-form-advance")
-
-
-async def _is_autocomplete_commit_required_enabled(task: Task) -> bool:
- return await _is_org_flag_enabled(task, AUTOCOMPLETE_COMMIT_REQUIRED_FLAG, "autocomplete-commit-required")
-
-
-async def _is_input_text_commit_verification_enabled(task: Task) -> bool:
- return await _is_org_flag_enabled(task, VERIFY_INPUT_TEXT_COMMIT_FLAG, "input-text-commit")
-
-
async def _resolve_collapse_xp_assignment(
experimentation_provider: BaseExperimentationProvider,
task: Task,
@@ -1647,274 +1417,6 @@ async def _fill_nanp_tel_with_readback(
return None
-_INPUT_TEXT_MASK_FORMAT_RE = re.compile(r"[\s\-_().]")
-_NON_TEXT_INPUT_TYPES = {
- "button",
- "checkbox",
- "color",
- "date",
- "datetime-local",
- "file",
- "hidden",
- "image",
- "month",
- "password",
- "radio",
- "range",
- "reset",
- "submit",
- "tel",
- "time",
- "week",
-}
-_INPUT_COMMIT_NAVIGATION_ERRORS = (
- "execution context was destroyed",
- "navigation",
- "target closed",
-)
-_INPUT_COMMIT_DETACHED_ERRORS = ("detached", "not attached")
-_MASK_NORMALIZED_INPUT_TYPES = {"search", "text", "textarea"}
-_SET_NATIVE_INPUT_VALUE_JS = """
-(element, value) => {
- const prototype = element instanceof HTMLTextAreaElement
- ? HTMLTextAreaElement.prototype
- : HTMLInputElement.prototype;
- const setter = Object.getOwnPropertyDescriptor(prototype, "value").set;
- setter.call(element, value);
- element.dispatchEvent(new Event("input", { bubbles: true }));
- element.dispatchEvent(new Event("change", { bubbles: true }));
- element.blur();
-}
-"""
-
-
-def _normalize_input_text_value(value: str | None) -> str:
- return _INPUT_TEXT_MASK_FORMAT_RE.sub("", value or "")
-
-
-def input_text_values_match(expected_value: str, actual_value: str | None, *, input_type: str) -> bool:
- """Decide whether a field committed what was typed.
-
- A field has committed when its value derives from the typed text. That covers an exact match, a
- reformat that only rearranges separators ("1234567890" -> "(123) 456-7890"), and an enrichment
- that extends the typed value ("12345" -> "12345-6789"). Only three things are failures: an empty
- field, a field still showing unfilled mask placeholders, and a genuinely different value.
-
- Deriving is read as a prefix rather than a substring: extending what was typed is a commit, but a
- value that merely happens to contain it is not.
- """
- expected_trimmed = expected_value.strip()
- actual_trimmed = (actual_value or "").strip()
- if not expected_trimmed:
- return True
- if not actual_trimmed:
- return False
- if expected_trimmed == actual_trimmed:
- return True
- if input_type not in _MASK_NORMALIZED_INPUT_TYPES:
- return False
- expected_normalized = _normalize_input_text_value(expected_trimmed).casefold()
- actual_normalized = _normalize_input_text_value(actual_trimmed).casefold()
- if not expected_normalized:
- # Nothing but separators was typed, so only separators may come back - anything else is a
- # different value, not a reformat of this one.
- return not actual_normalized
- # An all-placeholder commit ("_____") normalizes away to nothing: the mask took the keystrokes
- # and kept none of them.
- if not actual_normalized:
- return False
- return actual_normalized.startswith(expected_normalized)
-
-
-def _is_input_commit_navigation_error(exc: PlaywrightError) -> bool:
- return any(message in str(exc).lower() for message in _INPUT_COMMIT_NAVIGATION_ERRORS)
-
-
-def _is_input_commit_detached_error(exc: PlaywrightError) -> bool:
- return any(message in str(exc).lower() for message in _INPUT_COMMIT_DETACHED_ERRORS)
-
-
-def _is_plausible_autocomplete_rewrite(expected_value: str, actual_value: str | None) -> bool:
- # A suggestion rewrite contains or extends the typed text; the reverse containment
- # (actual is a prefix/fragment of expected) is a dropped-input failure, not a rewrite.
- expected_trimmed = expected_value.strip().casefold()
- actual_trimmed = (actual_value or "").strip().casefold()
- return bool(expected_trimmed and actual_trimmed) and expected_trimmed in actual_trimmed
-
-
-_OPTION_UI_ROLES = {"listbox", "option"}
-
-
-def _incremental_elements_contain_option_ui(elements: list[dict]) -> bool:
- stack: list[dict] = [element for element in elements if isinstance(element, dict)]
- while stack:
- element = stack.pop()
- if str(element.get("tagName") or "").lower() == "li":
- return True
- attributes = element.get("attributes")
- if isinstance(attributes, dict) and str(attributes.get("role") or "").lower() in _OPTION_UI_ROLES:
- return True
- children = element.get("children")
- if isinstance(children, list):
- stack.extend(child for child in children if isinstance(child, dict))
- return False
-
-
-async def _read_generic_input_commit(
- *,
- skyvern_element: SkyvernElement,
- dom: DomUtil,
- tag_name: str,
- input_type: str,
- expected_value: str,
- allow_autocomplete_rewrite: bool,
- allow_reresolve: bool,
-) -> tuple[bool, str | None, SkyvernElement]:
- try:
- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())
- except PlaywrightError as exc:
- if _is_input_commit_navigation_error(exc):
- return True, None, skyvern_element
- if not allow_reresolve or not _is_input_commit_detached_error(exc):
- raise
- skyvern_element = await dom.get_skyvern_element_by_id(skyvern_element.get_id())
- actual_value = await get_input_value(tag_name=tag_name, locator=skyvern_element.get_locator())
- committed = input_text_values_match(expected_value, actual_value, input_type=input_type)
- if not committed and allow_autocomplete_rewrite:
- committed = _is_plausible_autocomplete_rewrite(expected_value, actual_value)
- return committed, actual_value, skyvern_element
-
-
-async def _verify_generic_input_commit_with_escalation(
- *,
- skyvern_element: SkyvernElement,
- skyvern_frame: SkyvernFrame,
- dom: DomUtil,
- tag_name: str,
- input_type: str,
- expected_value: str,
- allow_autocomplete_rewrite: bool,
-) -> ActionFailure | None:
- # React-style inputs re-render nodes on every value change, so each escalation stage
- # gets its own re-resolve allowance instead of one shared across the ladder.
- committed, actual_value, skyvern_element = await _read_generic_input_commit(
- skyvern_element=skyvern_element,
- dom=dom,
- tag_name=tag_name,
- input_type=input_type,
- expected_value=expected_value,
- allow_autocomplete_rewrite=allow_autocomplete_rewrite,
- allow_reresolve=True,
- )
- if committed:
- return None
-
- # A masked or debounced field can land its value a beat after the keystrokes settle. Let it
- # before escalating, or the clear+retype below destroys a value that was about to commit.
- await skyvern_frame.safe_wait_for_animation_end(caller="input_text.commit_readback")
- committed, actual_value, skyvern_element = await _read_generic_input_commit(
- skyvern_element=skyvern_element,
- dom=dom,
- tag_name=tag_name,
- input_type=input_type,
- expected_value=expected_value,
- allow_autocomplete_rewrite=allow_autocomplete_rewrite,
- allow_reresolve=True,
- )
- if committed:
- return None
-
- await skyvern_element.input_clear()
- await skyvern_element.input_sequentially(text=expected_value)
- committed, actual_value, skyvern_element = await _read_generic_input_commit(
- skyvern_element=skyvern_element,
- dom=dom,
- tag_name=tag_name,
- input_type=input_type,
- expected_value=expected_value,
- allow_autocomplete_rewrite=allow_autocomplete_rewrite,
- allow_reresolve=True,
- )
- if committed:
- return None
-
- await skyvern_element.input_fill(text=expected_value)
- committed, actual_value, skyvern_element = await _read_generic_input_commit(
- skyvern_element=skyvern_element,
- dom=dom,
- tag_name=tag_name,
- input_type=input_type,
- expected_value=expected_value,
- allow_autocomplete_rewrite=allow_autocomplete_rewrite,
- allow_reresolve=True,
- )
- if committed:
- return None
-
- await skyvern_element.get_locator().evaluate(_SET_NATIVE_INPUT_VALUE_JS, expected_value)
- committed, actual_value, skyvern_element = await _read_generic_input_commit(
- skyvern_element=skyvern_element,
- dom=dom,
- tag_name=tag_name,
- input_type=input_type,
- expected_value=expected_value,
- allow_autocomplete_rewrite=allow_autocomplete_rewrite,
- allow_reresolve=True,
- )
- if committed:
- return None
-
- expected_length = len(_normalize_input_text_value(expected_value))
- actual_length = len(_normalize_input_text_value(actual_value))
- LOG.warning(
- "Input text commit mismatch after fallback ladder",
- element_id=skyvern_element.get_id(),
- expected_length=expected_length,
- actual_length=actual_length,
- )
- return ActionFailure(InputTextCommitMismatch(expected_length=expected_length, actual_length=actual_length))
-
-
-async def _verify_generic_input_commit(
- *,
- skyvern_element: SkyvernElement,
- skyvern_frame: SkyvernFrame,
- dom: DomUtil,
- tag_name: str,
- input_type: str,
- expected_value: str,
- allow_autocomplete_rewrite: bool = False,
-) -> ActionFailure | None:
- try:
- return await _verify_generic_input_commit_with_escalation(
- skyvern_element=skyvern_element,
- skyvern_frame=skyvern_frame,
- dom=dom,
- tag_name=tag_name,
- input_type=input_type,
- expected_value=expected_value,
- allow_autocomplete_rewrite=allow_autocomplete_rewrite,
- )
- except PlaywrightError as exc:
- if _is_input_commit_navigation_error(exc):
- return None
- if _is_input_commit_detached_error(exc):
- LOG.info(
- "Element detached during input commit verification; skipping verification",
- element_id=skyvern_element.get_id(),
- )
- return None
- raise
-
-
-def _is_generic_input_commit_candidate(*, tag_name: str, live_input_type: str) -> str | None:
- if tag_name == "textarea":
- return "textarea"
- if tag_name != InteractiveElement.INPUT:
- return None
- return live_input_type if live_input_type not in _NON_TEXT_INPUT_TYPES else None
-
-
async def _log_tel_fallback_fill_digit_counts(
*, skyvern_element: SkyvernElement, tag_name: str, expected_value: str, task_id: str | None, step_id: str | None
) -> None:
@@ -2058,66 +1560,6 @@ async def _fill_card_number_with_readback(
)
-# Character an input-mask library leaves in unfilled slots (jQuery Inputmask, Cleave, and component-library
-# masked inputs generally).
-_MASK_PLACEHOLDER_CHAR = "_"
-
-
-def _alnum(value: str | None) -> str:
- return "".join(ch for ch in (value or "") if ch.isalnum())
-
-
-def _has_input_mask_evidence(*, pattern: str | None, placeholder: str | None) -> bool:
- # Only treat a field as masked when it advertises a constraint: an HTML `pattern`, or a placeholder
- # rendered from mask slots (e.g. "_____", "___-__-____"). Keeps unrelated inputs untouched.
- if pattern:
- return True
- return placeholder is not None and _MASK_PLACEHOLDER_CHAR in placeholder
-
-
-def _committed_value_is_masked_incomplete(expected_value: str, committed_value: str | None) -> bool:
- # The mask accepted keystrokes but did not commit them: reads back empty (only separators), or still
- # shows unfilled placeholder slots while holding fewer characters than were typed. Compares character
- # counts only — never the raw value, which may be a secret. A placeholder char inside an otherwise
- # complete value (e.g. "foo_bar") is not treated as incomplete.
- expected_alnum = _alnum(expected_value)
- if not expected_alnum:
- return False
- committed_alnum = _alnum(committed_value)
- if not committed_alnum:
- return True
- return (
- committed_value is not None
- and _MASK_PLACEHOLDER_CHAR in committed_value
- and len(committed_alnum) < len(expected_alnum)
- )
-
-
-async def _verify_masked_input_after_fill(
- *,
- skyvern_element: SkyvernElement,
- skyvern_frame: SkyvernFrame,
- tag_name: str,
- expected_value: str,
- known_masked: bool,
-) -> None:
- committed_value = await get_input_value(tag_name, skyvern_element.get_locator())
- triggered = known_masked or (committed_value is not None and _MASK_PLACEHOLDER_CHAR in committed_value)
- if not triggered or not _committed_value_is_masked_incomplete(expected_value, committed_value):
- return
-
- # A masked value can land a beat after the keystrokes settle; let the field commit before verdict.
- await skyvern_frame.safe_wait_for_animation_end(caller="input_text.masked_readback")
- committed_value = await get_input_value(tag_name, skyvern_element.get_locator())
- if not _committed_value_is_masked_incomplete(expected_value, committed_value):
- return
-
- raise MaskedInputReadbackMismatch(
- expected_char_count=len(_alnum(expected_value)),
- committed_char_count=len(_alnum(committed_value)),
- )
-
-
# Native input types whose DOM .value round-trips typed text exactly, so an exact read-back is meaningful.
# password/text/email/search/url and an untyped input default to a text field; tel/number/date-like inputs
# normalize or auto-format their value and are excluded, as are textarea/contenteditable/select (non-input
@@ -2507,12 +1949,7 @@ async def check_date_format(
class AutoCompletionResult(BaseModel):
auto_completion_attempt: bool = False
incremental_elements: list[dict] = []
- action_result: ActionResult | None = None
- stage_outcome: str | None = None
- # A click landed and mutated the field but the commit could not be verified. Once this is
- # set the field is already dirty, so the caller must stop trying further selections instead
- # of clicking more options on top of it.
- committed_side_effects: bool = False
+ action_result: ActionResult = ActionSuccess()
class ScopedXhrDownloadCapture:
@@ -3300,91 +2737,6 @@ async def _retarget_disabled_element_for_click(
return child_element
-_SUBMIT_LIKE_TEXT = re.compile(
- r"\b(next|submit|continue|proceed|confirm|finish|save|apply|update|complete|register|sign\s?up|log\s?in)\b",
- re.IGNORECASE,
-)
-
-_BLOCKED_FORM_ADVANCE_MESSAGE = (
- "Clicked '{label}' but the page did not advance and no new elements appeared. The form is likely "
- "blocked by a validation error — inspect the visible fields for invalid or required inputs and any "
- "error messages, resolve them, and only then click to advance again instead of repeating this click."
-)
-
-
-def _submit_like_label(element_dict: dict) -> str:
- attributes = element_dict.get("attributes") or {}
- for candidate in (
- element_dict.get("text"),
- attributes.get("value"),
- attributes.get("aria-label"),
- attributes.get("title"),
- ):
- text = str(candidate or "").strip()
- if text:
- return text
- return "the button"
-
-
-def _click_is_submit_like(element_dict: dict) -> bool:
- attributes = element_dict.get("attributes") or {}
- tag = str(element_dict.get("tagName") or "").lower()
- input_type = str(attributes.get("type") or "").lower()
- if tag == "input" and input_type in {"submit", "image"}:
- return True
- role = str(attributes.get("role") or attributes.get("aria-role") or "").lower()
- # A bare <a> is a navigation link (a real one changes the URL and is excluded by the
- # caller's URL-change check); only an <a role="button"> counts as a submit control.
- is_button_like = tag == "button" or role == "button" or (tag == "input" and input_type == "button")
- if not is_button_like:
- return False
- label = " ".join(
- str(part)
- for part in (
- element_dict.get("text"),
- attributes.get("value"),
- attributes.get("aria-label"),
- attributes.get("title"),
- )
- if part
- )
- return bool(_SUBMIT_LIKE_TEXT.search(label))
-
-
-async def _maybe_flag_blocked_form_advance(
- *,
- results: list[ActionResult],
- skyvern_element: SkyvernElement,
- page: Page,
- original_url: str,
- incremental_scraped: IncrementalScrapePage,
- task: Task,
-) -> None:
- """Surface a diagnostic when a submit-like click neither navigated nor revealed new elements.
-
- The signal flows to the next step's planner through ActionResult.followup_message (see
- get_action_history), directing the agent to resolve the blocking validation instead of
- re-clicking a button that is not advancing the form.
- """
- if not results or not isinstance(results[-1], ActionSuccess):
- return
- element_dict = skyvern_element.get_element_dict()
- if not _click_is_submit_like(element_dict):
- return
- if page.url != original_url:
- return
- if not await _is_surface_blocked_form_advance_enabled(task):
- return
- if await incremental_scraped.get_incremental_elements_num() > 0:
- return
- results[-1].needs_followup = True
- results[-1].followup_message = _BLOCKED_FORM_ADVANCE_MESSAGE.format(label=_submit_like_label(element_dict))
- LOG.info(
- "Surfacing blocked-form-advance diagnostic after a non-advancing submit-like click",
- element_id=skyvern_element.get_id(),
- )
-
-
@traced(name="skyvern.agent.action.click")
async def handle_click_action(
action: actions.ClickAction,
@@ -3543,17 +2895,6 @@ async def handle_click_action(
results.append(sequential_click_result)
return results
- # No navigation and no dropdown surfaced: flag a non-advancing submit-like click so the
- # next step's planner resolves the blocking validation instead of re-clicking.
- await _maybe_flag_blocked_form_advance(
- results=results,
- skyvern_element=skyvern_element,
- page=page,
- original_url=original_url,
- incremental_scraped=incremental_scraped,
- task=task,
- )
-
except Exception:
LOG.warning(
"Failed to do sequential logic for the click action, skipping",
@@ -4100,7 +3441,6 @@ async def handle_input_text_action(
incremental_element: list[dict] = []
auto_complete_hacky_flag: bool = False
- autocomplete_option_signal: bool = False
input_or_select_context = await _get_input_or_select_context(
action=action,
@@ -4268,8 +3608,7 @@ async def handle_input_text_action(
)
return [ActionFailure(InputToReadonlyElement(element_id=skyvern_element.get_id()))]
- live_input_type = ((await skyvern_element.get_attr("type")) or "text").lower()
- is_tel = live_input_type == "tel"
+ is_tel = await skyvern_element.get_attr("type") == "tel"
candidate_card_digits = _card_number_digits(text)
is_card_number_input = _is_probable_card_number(candidate_card_digits) and await _is_card_number_field(
skyvern_element
@@ -4375,11 +3714,8 @@ async def handle_input_text_action(
"Find a blocking element to the current element, going to input on the blocking element",
)
if await blocking_element.is_editable():
- blocking_tag_name = blocking_element.get_tag_name()
- blocking_input_type = ((await blocking_element.get_attr("type")) or "text").lower()
skyvern_element = blocking_element
- tag_name = blocking_tag_name
- live_input_type = blocking_input_type
+ tag_name = blocking_element.get_tag_name()
if used_bare_nanp:
# The tel plan read constraints from the original element; re-derive them from
# the element actually being filled.
@@ -4438,7 +3774,7 @@ async def handle_input_text_action(
if len(text) == 0:
return [ActionSuccess()]
- if tag_name == InteractiveElement.INPUT and live_input_type == "date":
+ if tag_name == InteractiveElement.INPUT and await skyvern_element.get_attr("type") == "date":
try:
action.set_has_mini_agent()
text = await check_date_format(
@@ -4512,32 +3848,7 @@ async def handle_input_text_action(
and secret_input_type in _EXACT_VALUE_INPUT_TYPES
)
secret_maxlength = await skyvern_element.get_attr("maxlength") if verify_secret_input else None
-
- # Masked inputs (postal code, SSN) can report a successful fill while the mask committed nothing.
- # Read the value back for masked fields and, on an empty/placeholder commit, retype once. Scoped to
- # non-credential fields: a credential value is handled by the exact secret read-back above.
- verify_masked_input_after_fill = False
- mask_attr_evidence = False
- if not is_tel and not verify_secret_input and await _is_masked_input_readback_fix_enabled(task):
- mask_attr_evidence = _has_input_mask_evidence(
- pattern=await skyvern_element.get_attr("pattern"),
- placeholder=await skyvern_element.get_attr("placeholder"),
- )
- verify_masked_input_after_fill = True
-
- generic_commit_input_type = None
- if (
- not is_tel
- and not card_expected_digits
- and not verify_secret_input
- and await _is_input_text_commit_verification_enabled(task)
- ):
- generic_commit_input_type = _is_generic_input_commit_candidate(
- tag_name=tag_name,
- live_input_type=live_input_type,
- )
await incremental_scraped.start_listen_dom_increment(await skyvern_element.get_element_handler())
- input_context_destroyed = False
try:
# Read-back/recover priority: card > tel > secret > masked > plain fill (each gate is mutually exclusive).
@@ -4577,41 +3888,6 @@ async def handle_input_text_action(
)
if secret_failure is not None:
return [secret_failure]
- elif verify_masked_input_after_fill:
- await skyvern_element.input_sequentially(text=text)
- # Read the committed value back; on an empty/placeholder commit, refocus, clear and retype
- # once. A second failure fails the action with a diagnostic rather than a silent empty fill.
- try:
- await _verify_masked_input_after_fill(
- skyvern_element=skyvern_element,
- skyvern_frame=skyvern_frame,
- tag_name=tag_name,
- expected_value=text,
- known_masked=mask_attr_evidence,
- )
- except MaskedInputReadbackMismatch:
- await skyvern_element.get_locator().focus(timeout=timeout)
- await skyvern_element.input_clear()
- await skyvern_element.input_sequentially(text=text)
- try:
- # Attempt 1 only raises once the field is proven masked, so the retry keeps that
- # verdict: a mask that swallows the retype reads back clean of placeholder chars.
- await _verify_masked_input_after_fill(
- skyvern_element=skyvern_element,
- skyvern_frame=skyvern_frame,
- tag_name=tag_name,
- expected_value=text,
- known_masked=True,
- )
- except MaskedInputReadbackMismatch as mismatch:
- LOG.warning(
- "Masked input read-back mismatch after retry",
- action=action,
- element_id=skyvern_element.get_id(),
- expected_char_count=mismatch.expected_char_count,
- committed_char_count=mismatch.committed_char_count,
- )
- return [ActionFailure(mismatch)]
else:
await skyvern_element.input_sequentially(text=text)
if log_tel_fallback_readback:
@@ -4632,7 +3908,6 @@ async def handle_input_text_action(
)
if len(incremental_element) > 0:
auto_complete_hacky_flag = True
- autocomplete_option_signal = _incremental_elements_contain_option_ui(incremental_element)
if (
input_or_select_context
and input_or_select_context.is_search_bar
@@ -4641,6 +3916,7 @@ async def handle_input_text_action(
LOG.info(
"Detected target-matching dropdown after search-bar input; attempting custom selection",
element_id=skyvern_element.get_id(),
+ target_value=text,
)
action.set_has_mini_agent()
select_result = await sequentially_select_from_dropdown(
@@ -4710,7 +3986,6 @@ async def handle_input_text_action(
or "navigation" in error_message
or "target closed" in error_message
):
- input_context_destroyed = True
# These are expected during page navigation/auto-submit, silently continue
LOG.debug(
"Playwright error during incremental element processing (likely page navigation)",
@@ -4735,19 +4010,6 @@ async def handle_input_text_action(
# Always stop listening
await incremental_scraped.stop_listen_dom_increment()
- if generic_commit_input_type is not None and not input_context_destroyed:
- commit_failure = await _verify_generic_input_commit(
- skyvern_element=skyvern_element,
- skyvern_frame=skyvern_frame,
- dom=dom,
- tag_name=tag_name,
- input_type=generic_commit_input_type,
- expected_value=text,
- allow_autocomplete_rewrite=autocomplete_option_signal,
- )
- if commit_failure is not None:
- return [commit_failure]
-
return [ActionSuccess()]
except Exception as e:
# Handle any other unexpected errors during text input
@@ -6464,11 +5726,8 @@ async def choose_auto_completion_dropdown(
preserved_elements: list[dict] | None = None,
relevance_threshold: float = 0.8,
is_location_input: bool = False,
- commit_required: bool = False,
- verify_commit: bool = True,
collapse_autocomplete_fanout_enabled: bool = False,
action: InputTextAction | None = None,
- poll_deadline: float | None = None,
) -> AutoCompletionResult:
preserved_elements = preserved_elements or []
clear_input = True
@@ -6481,12 +5740,12 @@ async def choose_auto_completion_dropdown(
try:
await skyvern_element.press_fill(text)
- incremental_element = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scraped,
- cleanup_factory=clean_and_remove_element_tree_factory(
+ # wait for new elemnts to load
+ await skyvern_frame.safe_wait_for_animation_end(before_wait_sec=1, caller="autocomplete.fill")
+ incremental_element = await incremental_scraped.get_incremental_element_tree(
+ clean_and_remove_element_tree_factory(
task=task, step=step, check_filter_funcs=[check_existed_but_not_option_element_in_dom_factory(dom)]
),
- deadline=poll_deadline,
)
# check if elements in preserve list are still on the page
@@ -6548,7 +5807,7 @@ async def choose_auto_completion_dropdown(
"Autocomplete deterministic option identity failed, resetting input before LLM fallback",
element_id=matched_element_id,
matched_index=matched_index,
- matched_label_length=len(matched_label),
+ matched_label=matched_label,
)
(
incremental_scraped,
@@ -6566,7 +5825,6 @@ async def choose_auto_completion_dropdown(
text=text,
task=task,
step=step,
- poll_deadline=poll_deadline,
)
result.incremental_elements = copy.deepcopy(fallback_incremental_elements)
cleaned_incremental_element = shadow_candidate_elements
@@ -6574,45 +5832,32 @@ async def choose_auto_completion_dropdown(
LOG.info(
"Autocomplete deterministic fast path: exact/stem option found, skipping LLM",
element_id=matched_element_id,
- input_length=len(text),
+ input_value=text,
matched_index=matched_index,
- matched_label_length=len(matched_label),
+ matched_label=matched_label,
)
try:
await matched_locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
- # This deterministic read-back predates the commit-verification feature
- # (it is the one verify site that exists on main), so it stays on
- # regardless of verify_commit - gating it would reintroduce the silent
- # deterministic-click success this whole change exists to stop.
- commit_outcome = await _verify_autocomplete_input_readback(
+ if await _verify_autocomplete_input_readback(
skyvern_element=skyvern_element,
+ matched_index=matched_index,
matched_label=matched_label,
- typed_prefix=text,
- pre_action_value=text,
- suggestions_closed=await _autocomplete_suggestions_closed(matched_locator),
- selection_state_present=await _autocomplete_selection_state(matched_locator),
- commit_required=commit_required,
- )
- if commit_outcome.committed:
+ ):
clear_input = False
result.action_result = ActionSuccess()
return result
- result.stage_outcome = "clicked_but_not_committed"
- if commit_outcome.side_effects_observed:
- result.committed_side_effects = True
- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")
LOG.info(
"Autocomplete deterministic read-back failed, resetting input before LLM fallback",
element_id=matched_element_id,
matched_index=matched_index,
+ matched_label=matched_label,
)
- except AutoCompletionCommitFailure:
- raise
except Exception:
LOG.info(
"Autocomplete deterministic fast-path click/read-back failed, falling through to LLM",
element_id=matched_element_id,
matched_index=matched_index,
+ matched_label=matched_label,
exc_info=True,
)
(
@@ -6631,7 +5876,6 @@ async def choose_auto_completion_dropdown(
text=text,
task=task,
step=step,
- poll_deadline=poll_deadline,
)
result.incremental_elements = copy.deepcopy(fallback_incremental_elements)
cleaned_incremental_element = shadow_candidate_elements
@@ -6643,7 +5887,7 @@ async def choose_auto_completion_dropdown(
"Autocomplete deterministic option detached before click, resetting input before LLM fallback",
element_id=matched_element_id,
matched_index=matched_index,
- matched_label_length=len(matched_label),
+ matched_label=matched_label,
)
(
incremental_scraped,
@@ -6661,7 +5905,6 @@ async def choose_auto_completion_dropdown(
text=text,
task=task,
step=step,
- poll_deadline=poll_deadline,
)
result.incremental_elements = copy.deepcopy(fallback_incremental_elements)
cleaned_incremental_element = shadow_candidate_elements
@@ -6681,62 +5924,18 @@ async def choose_auto_completion_dropdown(
LOG.info(
"Location auto-completion fast path: single option found, skipping LLM",
element_id=fast_path_element_id,
- input_length=len(text),
+ input_value=text,
)
try:
await fast_path_locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS)
- if verify_commit:
- commit_outcome = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_element,
- matched_label=only_element.get("text") or "",
- typed_prefix=text,
- pre_action_value=text,
- suggestions_closed=await _autocomplete_suggestions_closed(fast_path_locator),
- selection_state_present=await _autocomplete_selection_state(fast_path_locator),
- commit_required=commit_required,
- )
- else:
- commit_outcome = AutocompleteCommitOutcome(committed=True)
- if commit_outcome.committed:
- clear_input = False
- result.action_result = ActionSuccess()
- return result
- result.stage_outcome = "clicked_but_not_committed"
- if commit_outcome.side_effects_observed:
- result.committed_side_effects = True
- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")
- LOG.info(
- "Location fast-path read-back failed, resetting input before LLM fallback",
- element_id=fast_path_element_id,
- )
- except AutoCompletionCommitFailure:
- raise
+ clear_input = False
+ result.action_result = ActionSuccess()
+ return result
except Exception:
LOG.info(
"Location fast-path click failed, falling through to LLM",
element_id=fast_path_element_id,
- exc_info=True,
)
- (
- incremental_scraped,
- fallback_incremental_elements,
- shadow_candidate_elements,
- html,
- new_interactable_element_ids,
- ) = await _reset_autocomplete_for_llm_fallback(
- current_incremental_scraped=incremental_scraped,
- skyvern_frame=skyvern_frame,
- skyvern_element=skyvern_element,
- page=page,
- scraped_page=scraped_page,
- dom=dom,
- text=text,
- task=task,
- step=step,
- poll_deadline=poll_deadline,
- )
- result.incremental_elements = copy.deepcopy(fallback_incremental_elements)
- cleaned_incremental_element = shadow_candidate_elements
if not html:
html = incremental_scraped.build_html_tree(cleaned_incremental_element)
@@ -6753,7 +5952,7 @@ async def choose_auto_completion_dropdown(
if (await dom_after_open.get_skyvern_element_by_id(element_id)).is_interactable()
]
if len(new_interactable_element_ids) == 0:
- raise AutoCompletionCommitFailure(stage="suggestions_never_rendered")
+ raise NoIncrementalElementFoundForAutoCompletion(element_id=skyvern_element.get_id(), text=text)
LOG.info(
"New elements detected after the input",
new_elements_ids=new_interactable_element_ids,
@@ -6800,31 +5999,11 @@ async def choose_auto_completion_dropdown(
if json_response.get("direct_searching", False):
LOG.info(
"Decided to directly search with the current value",
- element_id=skyvern_element.get_id(),
- input_length=len(text),
+ value=text,
)
await skyvern_element.press_key("Enter")
- matched_label = str(json_response.get("value") or "")
- # Searching with the typed text leaves the value unchanged by design, so an unchanged
- # read-back is a commit here - but not on a field that must commit a real suggestion,
- # where it is indistinguishable from Enter doing nothing at all.
- if verify_commit:
- commit_outcome = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_element,
- matched_label=matched_label or text,
- typed_prefix=text,
- pre_action_value=text,
- allow_unchanged_value=not commit_required,
- commit_required=commit_required,
- )
- else:
- commit_outcome = AutocompleteCommitOutcome(committed=True)
- if commit_outcome.committed:
- clear_input = False
- result.action_result = ActionSuccess()
- return result
- result.committed_side_effects = commit_outcome.side_effects_observed
- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")
+ clear_input = False
+ return result
if not element_id:
reasoning = json_response.get("reasoning")
@@ -6853,15 +6032,6 @@ async def choose_auto_completion_dropdown(
if await locator.count() == 0:
raise MissingElement(element_id=element_id)
- candidate_match = _autocomplete_candidate_for_element_id(shadow_candidate_elements, element_id)
- if candidate_match is None:
- raise NoSuitableAutoCompleteOption(
- reasoning="Selected element was not a rendered suggestion",
- target_value=text,
- )
- _matched_index, candidate = candidate_match
- matched_label = candidate.get("label") or ""
-
# Use SkyvernElement.click() so we get the full fallback chain
# (Playwright click → coordinate click → JavaScript click). Plain
# locator.click() can fail when the item or one of its ancestors has
@@ -6873,36 +6043,16 @@ async def choose_auto_completion_dropdown(
)
await selected_element.scroll_into_view()
await selected_element.click(page=page)
- if verify_commit:
- commit_outcome = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_element,
- matched_label=matched_label,
- typed_prefix=text,
- pre_action_value=text,
- suggestions_closed=await _autocomplete_suggestions_closed(locator),
- selection_state_present=await _autocomplete_selection_state(locator),
- commit_required=commit_required,
- )
- else:
- commit_outcome = AutocompleteCommitOutcome(committed=True)
- if commit_outcome.committed:
- clear_input = False
- result.action_result = ActionSuccess()
- return result
- result.committed_side_effects = commit_outcome.side_effects_observed
- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")
+ clear_input = False
+ return result
except Exception as e:
- stage = e.stage if isinstance(e, AutoCompletionCommitFailure) else "suggestion_not_matched"
LOG.info(
"Failed to choose the auto completion dropdown",
sampling=True,
- element_id=skyvern_element.get_id(),
- input_length=len(text),
- stage=stage,
exc_info=True,
+ input_value=text,
)
- result.stage_outcome = stage
result.action_result = ActionFailure(exception=e)
return result
finally:
@@ -6923,21 +6073,6 @@ def remove_duplicated_HTML_element(elements: list[dict]) -> list[dict]:
return new_elements
-def _autocomplete_side_effect_terminal(
- *, commit_required: bool, attempt_trail: list[str], element_id: str
-) -> ActionResult | None:
- LOG.info(
- "Autocomplete click mutated the field without a verified commit; stopping to avoid clicking further options on a dirty field",
- element_id=element_id,
- attempt_trail=attempt_trail,
- )
- if commit_required:
- return ActionFailure(
- AutoCompletionCommitFailure(stage="clicked_but_not_committed", attempt_trail=attempt_trail)
- )
- return None
-
-
async def input_or_auto_complete_input(
input_or_select_context: InputOrSelectContext,
scraped_page: ScrapedPage,
@@ -6965,16 +6100,6 @@ async def input_or_auto_complete_input(
current_attemp = 0
current_value = text
result = AutoCompletionResult()
- attempt_trail: list[str] = []
- is_location = input_or_select_context.is_location_input or False
- # Commit verification (and everything downstream of a failed read-back) only runs behind the
- # flag, so flag-off orgs keep the pre-verification behavior: a selection click is assumed
- # committed, with no read-back cost and no extra LLM fall-through.
- verify_commit = await _is_autocomplete_commit_required_enabled(task)
- # A location-search widget can be both. Search bars keep their documented fall-through, so they
- # never enter strict commit mode - otherwise the search-bar branch below returns a hard failure.
- commit_required = is_location and not input_or_select_context.is_search_bar and verify_commit
- poll_deadline = asyncio.get_running_loop().time() + AUTOCOMPLETE_TOTAL_DEADLINE_SECONDS
while current_attemp < MAX_AUTO_COMPLETE_ATTEMP:
current_attemp += 1
@@ -6984,9 +6109,9 @@ async def input_or_auto_complete_input(
LOG.info(
"Try the potential value for auto completion",
sampling=True,
- attempt=current_attemp,
- input_length=len(current_value),
+ input_value=current_value,
)
+ is_location = input_or_select_context.is_location_input or False
result = await choose_auto_completion_dropdown(
context=input_or_select_context,
page=page,
@@ -6998,30 +6123,19 @@ async def input_or_auto_complete_input(
step=step,
task=task,
is_location_input=is_location,
- commit_required=commit_required,
- verify_commit=verify_commit,
collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,
action=action,
- poll_deadline=poll_deadline,
)
if isinstance(result.action_result, ActionSuccess):
return ActionSuccess()
- attempt_trail.append(f"attempt_{len(attempt_trail) + 1}:{result.stage_outcome or 'suggestion_not_matched'}")
if input_or_select_context.is_search_bar:
- # commit_required is derived with `and not is_search_bar`, so it is always False here;
- # search bars keep their documented fall-through.
LOG.info(
"Stop generating potential values for the auto-completion since it's a search bar",
context=input_or_select_context,
)
return None
- if result.committed_side_effects:
- return _autocomplete_side_effect_terminal(
- commit_required=commit_required, attempt_trail=attempt_trail, element_id=skyvern_element.get_id()
- )
-
tried_values.append(current_value)
whole_new_elements.extend(result.incremental_elements)
@@ -7043,7 +6157,7 @@ async def input_or_auto_complete_input(
LOG.info(
"Ask LLM to give potential values based on the current value",
- input_length=len(current_value),
+ current_value=current_value,
potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,
)
if collapse_autocomplete_fanout_enabled and action is not None:
@@ -7058,14 +6172,13 @@ async def input_or_auto_complete_input(
if not value:
LOG.info(
"Empty potential value, skip this attempt",
- attempt=current_attemp,
+ value=each_value,
)
continue
LOG.info(
"Try the potential value for auto completion",
sampling=True,
- attempt=current_attemp,
- input_length=len(value),
+ input_value=value,
)
result = await choose_auto_completion_dropdown(
context=input_or_select_context,
@@ -7078,20 +6191,11 @@ async def input_or_auto_complete_input(
step=step,
task=task,
is_location_input=is_location,
- commit_required=commit_required,
- verify_commit=verify_commit,
collapse_autocomplete_fanout_enabled=collapse_autocomplete_fanout_enabled,
action=action,
- poll_deadline=poll_deadline,
)
if isinstance(result.action_result, ActionSuccess):
return ActionSuccess()
- attempt_trail.append(f"attempt_{len(attempt_trail) + 1}:{result.stage_outcome or 'suggestion_not_matched'}")
-
- if result.committed_side_effects:
- return _autocomplete_side_effect_terminal(
- commit_required=commit_required, attempt_trail=attempt_trail, element_id=skyvern_element.get_id()
- )
tried_values.append(value)
whole_new_elements.extend(result.incremental_elements)
@@ -7100,8 +6204,8 @@ async def input_or_auto_complete_input(
if current_attemp < MAX_AUTO_COMPLETE_ATTEMP:
LOG.info(
"Ask LLM to tweak the current value based on tried input values",
+ current_value=current_value,
current_attemp=current_attemp,
- input_length=len(current_value),
)
cleaned_new_elements = remove_duplicated_HTML_element(whole_new_elements)
prompt = prompt_engine.load_prompt(
@@ -7120,15 +6224,12 @@ async def input_or_auto_complete_input(
context_reasoning = json_respone.get("reasoning")
new_current_value = json_respone.get("tweaked_value", "")
if not new_current_value:
- if commit_required:
- stage = result.stage_outcome or "suggestion_not_matched"
- return ActionFailure(AutoCompletionCommitFailure(stage=stage, attempt_trail=attempt_trail))
return ActionFailure(ErrEmptyTweakValue(reasoning=context_reasoning, current_value=current_value))
LOG.info(
"Ask LLM tweaked the current value with a new value",
field_information=input_or_select_context.field,
- previous_length=len(current_value),
- new_length=len(new_current_value),
+ current_value=current_value,
+ new_value=new_current_value,
)
current_value = new_current_value
@@ -7137,38 +6238,26 @@ async def input_or_auto_complete_input(
LOG.info(
"Auto completion attempts exhausted, trying discover-all-options fallback",
element_id=skyvern_element.get_id(),
- input_length=len(text),
+ original_text=text,
)
- try:
- fallback_result = await discover_and_select_from_full_dropdown(
- context=input_or_select_context,
- page=page,
- scraped_page=scraped_page,
- dom=dom,
- original_text=text,
- skyvern_element=skyvern_element,
- step=step,
- task=task,
- commit_required=commit_required,
- verify_commit=verify_commit,
- )
- except AutoCompletionCommitFailure as fallback_failure:
- fallback_stage = fallback_failure.stage
- else:
- if isinstance(fallback_result, ActionSuccess):
- return fallback_result
- fallback_stage = "suggestion_not_matched"
- attempt_trail.append(f"attempt_{len(attempt_trail) + 1}:{fallback_stage}")
+ fallback_result = await discover_and_select_from_full_dropdown(
+ context=input_or_select_context,
+ page=page,
+ scraped_page=scraped_page,
+ dom=dom,
+ original_text=text,
+ skyvern_element=skyvern_element,
+ step=step,
+ task=task,
+ )
+ if fallback_result is not None:
+ return fallback_result
LOG.info(
"Auto completion didn't finish, this might leave the input value to be empty.",
sampling=True,
context=input_or_select_context,
- attempt_trail=attempt_trail,
)
- if commit_required:
- final_stage = attempt_trail[-1].split(":", 1)[1] if attempt_trail else "suggestion_not_matched"
- return ActionFailure(AutoCompletionCommitFailure(stage=final_stage, attempt_trail=attempt_trail))
return None
@@ -7183,8 +6272,6 @@ async def discover_and_select_from_full_dropdown(
step: Step,
task: Task,
relevance_threshold: float = 0.6,
- commit_required: bool = False,
- verify_commit: bool = True,
) -> ActionResult | None:
"""Fallback for auto-completion: clear input, click/ArrowDown to reveal all options,
then ask LLM to pick the best semantic match from actual dropdown values."""
@@ -7293,7 +6380,7 @@ async def discover_and_select_from_full_dropdown(
LOG.info(
"Discover fallback: asking LLM to pick from actual options",
element_id=skyvern_element.get_id(),
- input_length=len(original_text),
+ original_text=original_text,
)
json_response = await app.AUTO_COMPLETION_LLM_API_HANDLER(
prompt=prompt, step=step, prompt_name="auto-completion-choose-option"
@@ -7316,7 +6403,7 @@ async def discover_and_select_from_full_dropdown(
"Discover fallback: found suitable option, typing discovered value to trigger auto-completion",
element_id=element_id,
relevance_float=relevance_float,
- discovered_value_length=len(discovered_value),
+ discovered_value=discovered_value,
)
if not discovered_value:
@@ -7337,48 +6424,24 @@ async def discover_and_select_from_full_dropdown(
try:
await skyvern_element.press_key("ArrowDown")
await skyvern_element.press_key("Enter")
- # This path fills the field with the option text itself, so a widget that commits that text
- # verbatim reads back identical to the fill and the value alone proves nothing. aria-expanded
- # is the tiebreak: closed means Enter took, open means it did not. A widget that exposes no
- # expanded state at all - the shadow-DOM case this fallback exists for - leaves the verbatim
- # read-back as the only evidence there is, so accept it rather than fail a committed field.
- if verify_commit:
- suggestions_closed = await _autocomplete_input_reports_closed(skyvern_element)
- commit_outcome = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_element,
- matched_label=discovered_value,
- typed_prefix=discovered_value,
- pre_action_value=discovered_value,
- suggestions_closed=suggestions_closed,
- allow_unchanged_value=suggestions_closed is None,
- commit_required=commit_required,
- )
- else:
- commit_outcome = AutocompleteCommitOutcome(committed=True)
- if commit_outcome.committed:
- LOG.info(
- "Discover fallback: selected option via keyboard",
- element_id=skyvern_element.get_id(),
- )
- return ActionSuccess()
- raise AutoCompletionCommitFailure(stage="clicked_but_not_committed")
- except AutoCompletionCommitFailure:
- raise
+ LOG.info(
+ "Discover fallback: selected option via keyboard",
+ discovered_value=discovered_value,
+ )
+ return ActionSuccess()
except Exception:
LOG.info(
"Discover fallback: keyboard selection failed",
- discovered_value_length=len(discovered_value),
exc_info=True,
+ discovered_value=discovered_value,
)
return None
- except AutoCompletionCommitFailure:
- raise
except Exception:
LOG.warning(
"Discover fallback failed",
- input_length=len(original_text),
exc_info=True,
+ original_text=original_text,
)
return None
finally:
@@ -8356,74 +7419,6 @@ def _collect_new_roots(element: dict, new_ids: set[str], out: list[dict]) -> Non
_collect_new_roots(child, new_ids, out)
-def _custom_select_fallback_subtrees(elements: list[dict], current_element_id: str) -> list[dict]:
- anchor: dict | None = None
- anchor_is_open = False
- listboxes: list[dict] = []
- anchor_descendant_listboxes: list[dict] = []
- all_nodes: list[dict] = []
-
- queue: deque[tuple[dict, bool, bool]] = deque((element, False, False) for element in elements)
- while queue:
- node, in_anchor, in_expanded_combobox = queue.popleft()
- if not isinstance(node, dict):
- continue
- attrs = node.get("attributes") or {}
- all_nodes.append(node)
- role = str(attrs.get("role") or "").lower()
- is_anchor = node.get("id") == current_element_id
- child_in_anchor = in_anchor or is_anchor
- expanded = str(attrs.get("aria-expanded") or "").lower() == "true"
- child_in_expanded_combobox = in_expanded_combobox or (role == "combobox" and expanded)
-
- if is_anchor:
- anchor = node
- anchor_is_open = expanded or in_expanded_combobox
- if role == "listbox":
- listboxes.append(node)
- if child_in_anchor:
- anchor_descendant_listboxes.append(node)
-
- for child in node.get("children") or []:
- queue.append((child, child_in_anchor, child_in_expanded_combobox))
-
- if anchor is None:
- return []
-
- anchor_attrs = anchor.get("attributes") or {}
- controlled_dom_ids = {
- reference
- for attribute_name in ("aria-controls", "aria-owns")
- for reference in str(anchor_attrs.get(attribute_name) or "").split()
- if reference
- }
- anchor_dom_id = str(anchor_attrs.get("id") or "")
- controlled_subtrees = [
- node
- for node in all_nodes
- if str((node.get("attributes") or {}).get("id") or "") in controlled_dom_ids
- and _custom_select_candidates_from_elements([node])
- ]
- if controlled_subtrees:
- return controlled_subtrees
-
- linked_listboxes = []
- for listbox in listboxes:
- attrs = listbox.get("attributes") or {}
- listbox_dom_id = str(attrs.get("id") or "")
- labelled_by = set(str(attrs.get("aria-labelledby") or "").split())
- if listbox_dom_id in controlled_dom_ids or (anchor_dom_id and anchor_dom_id in labelled_by):
- linked_listboxes.append(listbox)
-
- if linked_listboxes:
- return linked_listboxes
- if anchor_descendant_listboxes:
- return anchor_descendant_listboxes
- if len(listboxes) == 1 and anchor_is_open:
- return listboxes
- return []
-
-
@traced(name="skyvern.agent.dropdown.select_emerging")
async def select_from_emerging_elements(
current_element_id: str,
@@ -8453,44 +7448,12 @@ async def select_from_emerging_elements(
if (await dom_after_open.get_skyvern_element_by_id(element_id)).is_interactable()
]
- fallback_element_subtrees: list[dict] = []
- if len(new_interactable_element_ids) == 0:
- # Resolve ownership from the untrimmed tree: trim_element_tree drops aria-controls /
- # aria-owns / aria-labelledby / the DOM id, which are exactly the linkage attributes this
- # fallback keys off, so resolving against the trimmed tree leaves it inert. Re-trim the
- # small resolved subtree set below so the prompt stays trimmed (no page-wide token cost).
- fallback_element_subtrees = _custom_select_fallback_subtrees(
- scraped_page_after_open.element_tree,
- current_element_id,
- )
- fallback_element_subtrees = trim_element_tree(copy.deepcopy(fallback_element_subtrees))
- fallback_candidates = _custom_select_candidates_from_elements(fallback_element_subtrees)
- fallback_candidate_ids: list[str] = []
- for candidate in fallback_candidates:
- fallback_id = candidate.get("element_id")
- if fallback_id is not None:
- fallback_candidate_ids.append(fallback_id)
- new_interactable_element_ids = [
- element_id
- for element_id in fallback_candidate_ids
- if (await dom_after_open.get_skyvern_element_by_id(element_id)).is_interactable()
- ]
- if new_interactable_element_ids:
- LOG.info(
- "Found current-page custom-select options after incremental detection returned no elements",
- current_element_id=current_element_id,
- fallback_option_count=len(new_interactable_element_ids),
- )
-
if len(new_interactable_element_ids) == 0:
raise NoIncrementalElementFoundForCustomSelection(element_id=current_element_id)
- interactable_element_ids = set(new_interactable_element_ids)
# Extract minimal subtrees rooted at new elements — avoids sending the full page DOM
# which gets truncated on large pages, losing portal-rendered dropdown items.
- new_element_subtrees = fallback_element_subtrees or _extract_new_subtrees(
- scraped_page_after_open.element_tree_trimmed, new_element_ids
- )
+ new_element_subtrees = _extract_new_subtrees(scraped_page_after_open.element_tree_trimmed, new_element_ids)
shadow_candidate_elements: list[dict] = []
_ctx = skyvern_context.current()
lean_enabled = bool(_ctx and _ctx.enable_lean_element_tree)
@@ -8533,20 +7496,9 @@ async def select_from_emerging_elements(
async def get_readback_scope_element() -> SkyvernElement | None:
return await dom_after_open.get_skyvern_element_by_id(current_element_id)
- verify_pick_enabled = await _is_verify_emerging_select_pick_enabled(task)
- deterministic_candidates = _custom_select_candidates_from_elements(shadow_candidate_elements)
- if verify_pick_enabled:
- # Restrict deterministic matching to the elements the dropdown actually surfaced. Gated:
- # custom-widget option nodes are often not flagged interactable, so flag off keeps main's
- # behavior (all candidates) and this ramps with the off-list rejection below.
- deterministic_candidates = [
- candidate
- for candidate in deterministic_candidates
- if candidate.get("element_id") in interactable_element_ids
- ]
deterministic_result = await _select_deterministic_custom_option(
target_value=options.target_value,
- get_option_candidates=lambda: deterministic_candidates,
+ get_option_candidates=lambda: _custom_select_candidates_from_elements(shadow_candidate_elements),
field_context=options.model_dump(),
page=page,
get_skyvern_element=dom_after_open.get_skyvern_element_by_id,
@@ -8599,20 +7551,7 @@ async def select_from_emerging_elements(
llm_value=value,
),
)
- anchor_input_pick = raw_action_type == ActionType.INPUT_TEXT.value and element_id == current_element_id
- # A pick outside the elements the dropdown actually surfaced is a wrong-field click that used to
- # proceed on a warning. Rejecting it turns those silent successes into loud failures, so it ramps
- # behind a flag rather than shipping to every org at once.
- off_list = element_id not in interactable_element_ids and not anchor_input_pick
- if off_list and not verify_pick_enabled:
- # Keep the pre-flag warning so the reject rate is measurable before ramping the flag on.
- LOG.warning(
- "Emerging-select pick is outside the surfaced interactable elements; proceeding because the verify flag is off",
- element_id=element_id,
- current_element_id=current_element_id,
- )
- off_list_pick = off_list and verify_pick_enabled
- if not element_id or raw_action_type not in (ActionType.CLICK.value, ActionType.INPUT_TEXT.value) or off_list_pick:
+ if not element_id or raw_action_type not in (ActionType.CLICK.value, ActionType.INPUT_TEXT.value):
raise _no_match_exception_for_dropdown(
reasoning=json_response.get("reasoning"),
target_value=options.target_value,
@@ -8621,6 +7560,14 @@ async def select_from_emerging_elements(
)
action_type = ActionType(raw_action_type)
+ new_ids_set = set(new_interactable_element_ids)
+ if element_id not in new_ids_set:
+ LOG.warning(
+ "custom-select returned element outside new_interactable_element_ids",
+ selected_element_id=element_id,
+ new_interactable_count=len(new_ids_set),
+ )
+
if value is not None and action_type == ActionType.INPUT_TEXT:
actual_value = get_actual_value_of_parameter_if_secret_with_task(task, value)
LOG.info(
diff --git a/skyvern/webeye/scraper/domUtils.js b/skyvern/webeye/scraper/domUtils.js
index 764f7da91..a3b3dbe1c 100644
--- a/skyvern/webeye/scraper/domUtils.js
+++ b/skyvern/webeye/scraper/domUtils.js
@@ -2875,160 +2875,6 @@ function asyncSleepFor(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
-const incrementalObserverAttributeFilter = [
- "class",
- "style",
- "hidden",
- "aria-expanded",
- "aria-hidden",
-];
-const maxIncrementalShadowRootDepth = 10;
-const maxIncrementalShadowRootCount = 200;
-const maxIncrementalShadowDiscoveryVisits = 4000;
-const incrementalShadowDiscoveryBudgetMs = 25;
-
-function getIncrementalObserverOptions() {
- return {
- attributes: true,
- attributeOldValue: true,
- childList: true,
- subtree: true,
- characterData: true,
- };
-}
-
-function getIncrementalOptionLikeElements(
- root,
- recurseObservedShadowRoots = false,
-) {
- const selector =
- '[role="listbox"], [role="option"], [role="listbox"] li, [role="combobox"] li';
- const elements = [];
- const pending = [root];
- const visited = new Set();
- while (pending.length > 0) {
- const currentRoot = pending.pop();
- if (!currentRoot || visited.has(currentRoot)) continue;
- visited.add(currentRoot);
- if (currentRoot instanceof Element && currentRoot.matches(selector)) {
- elements.push(currentRoot);
- }
- if (currentRoot.querySelectorAll) {
- elements.push(...currentRoot.querySelectorAll(selector));
- }
- if (!recurseObservedShadowRoots) continue;
- const possibleHosts =
- currentRoot instanceof Element
- ? [currentRoot, ...currentRoot.querySelectorAll("*")]
- : [...currentRoot.querySelectorAll("*")];
- for (const possibleHost of possibleHosts) {
- if (
- possibleHost.shadowRoot &&
- window.globalObservedShadowRoots.has(possibleHost.shadowRoot)
- ) {
- pending.push(possibleHost.shadowRoot);
- }
- }
- }
- return elements;
-}
-
-function isIncrementalOptionSubtreeVisible(element) {
- return (
- !element.hidden &&
- element.getAttribute("aria-hidden") !== "true" &&
- isElementVisible(element)
- );
-}
-
-function trackIncrementalOptionVisibility(root) {
- for (const optionLikeElement of getIncrementalOptionLikeElements(root)) {
- let currentElement = optionLikeElement;
- while (currentElement instanceof Element) {
- if (window.globalIncrementalVisibilityMap.has(currentElement)) break;
- window.globalIncrementalVisibilityMap.set(
- currentElement,
- isIncrementalOptionSubtreeVisible(currentElement),
- );
- if (currentElement.parentElement) {
- currentElement = currentElement.parentElement;
- continue;
- }
- const currentRoot = currentElement.getRootNode();
- currentElement =
- currentRoot instanceof ShadowRoot ? currentRoot.host : null;
- }
- }
-}
-
-function observeIncrementalTarget(target) {
- window.globalObserverForDOMIncrement.observe(
- target,
- getIncrementalObserverOptions(),
- );
- trackIncrementalOptionVisibility(target);
-}
-
-function observeIncrementalShadowRoot(shadowRoot, countTowardBudget = true) {
- if (window.globalObservedShadowRoots.has(shadowRoot)) return;
- window.globalObservedShadowRoots.add(shadowRoot);
- if (countTowardBudget) {
- window.globalObservedShadowRootCount++;
- }
- observeIncrementalTarget(shadowRoot);
-}
-
-function discoverAndObserveOpenShadowRoots(
- startNode,
- initialDepth = 0,
- deadline = Number.POSITIVE_INFINITY,
- countTowardBudget = true,
-) {
- const pending = [{ node: startNode, depth: initialDepth }];
- let currentIndex = 0;
- while (
- currentIndex < pending.length &&
- currentIndex < maxIncrementalShadowDiscoveryVisits &&
- (!countTowardBudget ||
- window.globalObservedShadowRootCount < maxIncrementalShadowRootCount) &&
- performance.now() < deadline
- ) {
- const { node, depth } = pending[currentIndex++];
- if (!node || depth > maxIncrementalShadowRootDepth) continue;
-
- if (
- node instanceof Element &&
- node.shadowRoot &&
- depth < maxIncrementalShadowRootDepth
- ) {
- const shadowRoot = node.shadowRoot;
- if (!window.globalObservedShadowRoots.has(shadowRoot)) {
- observeIncrementalShadowRoot(shadowRoot, countTowardBudget);
- }
- for (const child of shadowRoot.children) {
- pending.push({ node: child, depth: depth + 1 });
- }
- }
-
- if (node.children) {
- for (const child of node.children) {
- pending.push({ node: child, depth });
- }
- }
- }
-}
-
-function isNewlyVisibleIncrementalOptionSubtree(node) {
- const wasVisible = window.globalIncrementalVisibilityMap.get(node) ?? false;
- const isVisible = isIncrementalOptionSubtreeVisible(node);
- window.globalIncrementalVisibilityMap.set(node, isVisible);
- return (
- !wasVisible &&
- isVisible &&
- getIncrementalOptionLikeElements(node, true).length > 0
- );
-}
-
async function addIncrementalNodeToMap(parentNode, childrenNode) {
const maxParsedElement = 3000;
const maxElementToWait = 100;
@@ -3045,7 +2891,10 @@ async function addIncrementalNodeToMap(parentNode, childrenNode) {
if (window.globalListnerFlag) {
// calculate the depth of targetNode element for sorting
const depth = getElementDomDepth(parentNode);
- const parsedNodeTrees = [];
+ let newNodesTreeList = [];
+ if (window.globalDomDepthMap.has(depth)) {
+ newNodesTreeList = window.globalDomDepthMap.get(depth);
+ }
try {
for (const child of childrenNode) {
@@ -3063,17 +2912,13 @@ async function addIncrementalNodeToMap(parentNode, childrenNode) {
window.globalHoverStylesMap,
);
if (newNodeTree.length > 0) {
- parsedNodeTrees.push(...newNodeTree);
+ newNodesTreeList.push(...newNodeTree);
}
}
} catch (error) {
_jsConsoleError("Error building incremental element node:", error);
}
- const currentNodeTrees = window.globalDomDepthMap.get(depth) || [];
- window.globalDomDepthMap.set(depth, [
- ...currentNodeTrees,
- ...parsedNodeTrees,
- ]);
+ window.globalDomDepthMap.set(depth, newNodesTreeList);
}
await window.globalParsedElementCounter.add();
}
@@ -3083,29 +2928,12 @@ if (window.globalObserverForDOMIncrement === undefined) {
mutationsList,
observer,
) {
- const shadowDiscoveryDeadline =
- performance.now() + incrementalShadowDiscoveryBudgetMs;
// TODO: how to detect duplicated recreate element?
for (const mutation of mutationsList) {
- const node =
- mutation.target instanceof ShadowRoot
- ? mutation.target.host
- : mutation.target;
+ const node = mutation.target;
if (node.nodeType === Node.TEXT_NODE) continue;
const tagName = node.tagName?.toLowerCase();
- if (mutation.type === "childList") {
- for (const addedNode of mutation.addedNodes) {
- if (!(addedNode instanceof Element)) continue;
- trackIncrementalOptionVisibility(addedNode);
- discoverAndObserveOpenShadowRoots(
- addedNode,
- 0,
- shadowDiscoveryDeadline,
- );
- }
- }
-
// ignore unique_id change to avoid infinite loop about DOM changes
if (mutation.attributeName === "unique_id") continue;
@@ -3127,19 +2955,6 @@ if (window.globalObserverForDOMIncrement === undefined) {
// we detect the element based on the following rules
switch (mutation.type) {
case "attributes": {
- if (
- incrementalObserverAttributeFilter.includes(
- mutation.attributeName,
- ) &&
- isNewlyVisibleIncrementalOptionSubtree(node)
- ) {
- window.globalOneTimeIncrementElements.push({
- targetNode: node,
- newNodes: [node],
- });
- await addIncrementalNodeToMap(node, [node]);
- break;
- }
switch (mutation.attributeName) {
case "hidden": {
if (!node.hidden) {
@@ -3235,48 +3050,25 @@ async function startGlobalIncrementalObserver(element = null) {
window.globalOneTimeIncrementElements = [];
await getHoverStylesMap();
window.globalParsedElementCounter = new SafeCounter();
- window.globalIncrementalVisibilityMap = new WeakMap();
- window.globalObservedShadowRoots = new WeakSet();
- window.globalObservedShadowRootCount = 0;
- window.globalIncrementalObserverGeneration =
- (window.globalIncrementalObserverGeneration || 0) + 1;
window.globalObserverForDOMIncrement.takeRecords(); // cleanup the older data
+ window.globalObserverForDOMIncrement.observe(document.body, {
+ attributes: true,
+ attributeOldValue: true,
+ childList: true,
+ subtree: true,
+ characterData: true,
+ });
- let composedTreeElement = element;
- let composedTreeDepth = 0;
- const composedTreeRoots = [];
- while (
- composedTreeElement instanceof Element &&
- composedTreeDepth < maxIncrementalShadowRootDepth
- ) {
- const root = composedTreeElement.getRootNode();
- if (!(root instanceof ShadowRoot)) break;
- observeIncrementalShadowRoot(root, false);
- composedTreeRoots.push({ root, depth: composedTreeDepth });
- composedTreeElement = root.host;
- composedTreeDepth++;
- }
-
- const anchorDiscoveryDeadline = performance.now() + 50;
- for (const { root, depth } of composedTreeRoots) {
- discoverAndObserveOpenShadowRoots(root, depth, anchorDiscoveryDeadline);
+ // if the element is in shadow DOM, we need to observe the shadow DOM as well
+ if (element && element.getRootNode() instanceof ShadowRoot) {
+ window.globalObserverForDOMIncrement.observe(element.getRootNode(), {
+ attributes: true,
+ attributeOldValue: true,
+ childList: true,
+ subtree: true,
+ characterData: true,
+ });
}
-
- observeIncrementalTarget(document.body);
- const observerGeneration = window.globalIncrementalObserverGeneration;
- setTimeout(() => {
- if (
- !window.globalListnerFlag ||
- observerGeneration !== window.globalIncrementalObserverGeneration
- ) {
- return;
- }
- discoverAndObserveOpenShadowRoots(
- document.documentElement,
- 0,
- performance.now() + 50,
- );
- }, 0);
}
async function stopGlobalIncrementalObserver() {
diff --git a/tests/unit/test_agent_step_characterization.py b/tests/unit/test_agent_step_characterization.py
index 14a170b3a..cad0ad6d7 100644
--- a/tests/unit/test_agent_step_characterization.py
+++ b/tests/unit/test_agent_step_characterization.py
@@ -9,14 +9,13 @@ from zoneinfo import ZoneInfo
import pytest
-from skyvern.exceptions import NoTOTPVerificationCodeFound, RepeatedActionFailure
-from skyvern.forge.agent import ForgeAgent, StepPromptResult, _get_repeated_action_failure
+from skyvern.exceptions import NoTOTPVerificationCodeFound
+from skyvern.forge.agent import ForgeAgent, StepPromptResult
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
from skyvern.forge.sdk.models import Step, StepStatus
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.tasks import Task
-from skyvern.schemas.steps import AgentStepOutput
from skyvern.webeye.actions.action_types import ActionType
from skyvern.webeye.actions.actions import (
Action,
@@ -24,7 +23,6 @@ from skyvern.webeye.actions.actions import (
CompleteAction,
DownloadFileAction,
ExtractAction,
- InputTextAction,
WaitAction,
)
from skyvern.webeye.actions.models import DetailedAgentStepOutput
@@ -79,7 +77,6 @@ def make_agent_step_rig(
injected_actions: list[Action] | None = None,
task_overrides: dict[str, Any] | None = None,
disable_user_goal_check: bool = True,
- repeated_action_breaker: bool = False,
) -> AgentStepRig:
agent = ForgeAgent()
now = datetime.now(UTC)
@@ -151,8 +148,6 @@ def make_agent_step_rig(
async def _flag(flag_name: str, *_args, **_kwargs) -> bool:
if flag_name == "DISABLE_USER_GOAL_CHECK":
return disable_user_goal_check
- if flag_name == "REPEATED_ACTION_CIRCUIT_BREAKER":
- return repeated_action_breaker
return False
monkeypatch.setattr(
@@ -289,137 +284,6 @@ async def test_failed_action_marks_step_failed_and_skips_remaining(monkeypatch:
assert output.actions_and_results[0][1][0].success is False
-async def test_third_identical_failed_persisted_input_surfaces_repeated_action_failure(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- action = InputTextAction(element_id="framework-managed-input", text="private payload")
- handler = AsyncMock(
- return_value=[ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)]
- )
- rig = make_agent_step_rig(
- monkeypatch, parsed_actions=[action], action_handler=handler, repeated_action_breaker=True
- )
- previous_steps = []
- for order in range(2):
- previous_action = action.model_copy(update={"step_id": f"step-{order}", "step_order": order})
- persisted_output = AgentStepOutput.model_validate(
- AgentStepOutput(
- action_results=[ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)],
- actions_and_results=[
- (
- previous_action,
- [ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)],
- ),
- ],
- errors=[],
- ).model_dump(mode="json")
- )
- assert persisted_output.actions_and_results is not None
- persisted_action = persisted_output.actions_and_results[0][0]
- assert type(persisted_action) is Action
- assert persisted_action.action_type == ActionType.INPUT_TEXT
- assert persisted_action.element_id == "framework-managed-input"
- assert persisted_action.text == "private payload"
- previous_step = make_step(
- datetime.now(UTC),
- rig.task,
- step_id=f"step-{order}",
- status=StepStatus.failed,
- order=order,
- output=persisted_output,
- )
- previous_steps.append(previous_step)
- monkeypatch.setattr(
- "skyvern.forge.agent.app.DATABASE.tasks.get_task_steps",
- AsyncMock(return_value=previous_steps),
- )
-
- step, output = await rig.run()
-
- assert step.status == StepStatus.completed
- assert handler.await_count == 1
- assert output.actions_and_results is not None
- repeated_result = output.actions_and_results[-1][1][-1]
- assert repeated_result.exception_type == RepeatedActionFailure.__name__
- assert "framework-managed-input" in (repeated_result.exception_message or "")
- assert "3" in (repeated_result.exception_message or "")
- assert "Exception" in (repeated_result.exception_message or "")
- assert "private payload" not in (repeated_result.exception_message or "")
- assert repeated_result.stop_execution_on_failure is False
-
-
-async def test_repeat_ineligible_failure_does_not_load_task_history(monkeypatch: pytest.MonkeyPatch) -> None:
- action = WaitAction(seconds=1)
- history = AsyncMock()
- monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task_steps", history)
- rig = make_agent_step_rig(
- monkeypatch,
- parsed_actions=[action],
- action_handler=AsyncMock(return_value=[ActionFailure(Exception("failed"))]),
- )
-
- await rig.run()
-
- history.assert_not_awaited()
-
-
-async def test_repeated_action_breaker_disabled_adds_no_db_read_and_no_failure(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- """Flag off is today's behavior: the failure surfaces as itself and the task-steps history is never
- loaded, so a flag-off org pays nothing for the detector on the failing-action hot path."""
- action = InputTextAction(element_id="framework-managed-input", text="private payload")
- history = AsyncMock(return_value=[])
- monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task_steps", history)
- rig = make_agent_step_rig(
- monkeypatch,
- parsed_actions=[action, action.model_copy(), action.model_copy()],
- action_handler=AsyncMock(
- return_value=[ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)]
- ),
- repeated_action_breaker=False,
- )
-
- _, output = await rig.run()
-
- history.assert_not_awaited()
- for _action, results in output.actions_and_results:
- assert results[-1].exception_type == "Exception"
- assert results[-1].exception_type != RepeatedActionFailure.__name__
-
-
-async def test_repeated_action_history_loaded_at_most_once_per_step(monkeypatch: pytest.MonkeyPatch) -> None:
- action = InputTextAction(element_id="framework-managed-input", text="private payload")
- history = AsyncMock(return_value=[])
- monkeypatch.setattr("skyvern.forge.agent.app.DATABASE.tasks.get_task_steps", history)
- rig = make_agent_step_rig(
- monkeypatch,
- parsed_actions=[action, action.model_copy()],
- action_handler=AsyncMock(
- return_value=[ActionFailure(Exception("value did not commit"), stop_execution_on_failure=False)]
- ),
- repeated_action_breaker=True,
- )
-
- await rig.run()
-
- assert rig.action_handler.await_count == 2
- assert history.await_count == 1
-
-
-def test_repeated_action_failure_requires_matching_unsuccessful_attempts() -> None:
- first = InputTextAction(element_id="field", text="first")
- second = InputTextAction(element_id="field", text="second")
- failure = [ActionFailure(Exception("failed"))]
-
- assert _get_repeated_action_failure([(first, failure), (first, [ActionSuccess()]), (first, failure)]) is None
- assert _get_repeated_action_failure([(first, failure), (second, failure), (first, failure)]) is None
-
-
@pytest.mark.asyncio
async def test_failed_action_with_continue_flag_executes_remaining(monkeypatch: pytest.MonkeyPatch) -> None:
# Mirror of the skip case: a failure result that opts out of stopping the batch
diff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.py
index 3126078f6..bc7a8bc1e 100644
--- a/tests/unit/test_auto_completion_discover_fallback.py
+++ b/tests/unit/test_auto_completion_discover_fallback.py
@@ -13,7 +13,6 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from skyvern.exceptions import AutoCompletionCommitFailure
from skyvern.forge.sdk.models import StepStatus
from skyvern.webeye.actions.actions import InputOrSelectContext
from skyvern.webeye.actions.handler import (
@@ -71,10 +70,7 @@ def _mock_skyvern_element(frame: MagicMock | None = None) -> MagicMock:
# get_locator().click() needs to be async for the discover fallback's click call
mock_locator = MagicMock()
mock_locator.click = AsyncMock()
- mock_locator.input_value = AsyncMock(return_value="I do not wish to disclose")
el.get_locator.return_value = mock_locator
- el.get_tag_name.return_value = "input"
- el.get_attr = AsyncMock(return_value="false")
return el
@@ -99,7 +95,6 @@ def _mock_incremental_scrape(elements: list[dict]) -> MagicMock:
inc = MagicMock()
inc.start_listen_dom_increment = AsyncMock()
inc.stop_listen_dom_increment = AsyncMock()
- inc.get_incremental_elements_num = AsyncMock(return_value=len(elements))
inc.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(elements))
inc.build_html_tree.return_value = "<div>mocked options</div>"
inc.id_to_element_dict = {e["id"]: e for e in elements}
@@ -170,260 +165,6 @@ async def test_discover_fallback_succeeds_when_options_appear() -> None:
inc_scrape.stop_listen_dom_increment.assert_awaited()
-async def test_discover_fallback_noop_keyboard_selection_is_not_a_commit() -> None:
- """The input still holding the fallback's own fill with the listbox open is not commit evidence."""
- frame = _mock_frame(locator_count=1)
- skyvern_el = _mock_skyvern_element(frame)
- skyvern_el.get_attr = AsyncMock(return_value="true")
- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)
-
- llm_response = {
- "auto_completion_attempt": True,
- "relevance_float": 0.9,
- "id": "OPT3",
- "value": "I do not wish to disclose",
- "reasoning": "'I do not wish to disclose' matches 'Decline to Self Identify'",
- }
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
- ),
- patch(
- "skyvern.webeye.actions.handler.IncrementalScrapePage",
- return_value=inc_scrape,
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- with pytest.raises(AutoCompletionCommitFailure) as exc_info:
- await discover_and_select_from_full_dropdown(
- context=_make_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- original_text="Decline to Self Identify",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
- assert exc_info.value.stage == "clicked_but_not_committed"
- inc_scrape.stop_listen_dom_increment.assert_awaited()
-
-
-async def test_discover_fallback_assumes_commit_when_verification_disabled() -> None:
- """Flag off: the keyboard selection is assumed committed - same scenario the noop test above
- rejects with verification enabled."""
- frame = _mock_frame(locator_count=1)
- skyvern_el = _mock_skyvern_element(frame)
- skyvern_el.get_attr = AsyncMock(return_value="true")
- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)
-
- llm_response = {
- "auto_completion_attempt": True,
- "relevance_float": 0.9,
- "id": "OPT3",
- "value": "I do not wish to disclose",
- "reasoning": "'I do not wish to disclose' matches 'Decline to Self Identify'",
- }
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
- ),
- patch(
- "skyvern.webeye.actions.handler.IncrementalScrapePage",
- return_value=inc_scrape,
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await discover_and_select_from_full_dropdown(
- context=_make_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- original_text="Decline to Self Identify",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- verify_commit=False,
- )
-
- assert isinstance(result, ActionSuccess)
- inc_scrape.stop_listen_dom_increment.assert_awaited()
-
-
-async def test_discover_fallback_canonicalized_commit_succeeds() -> None:
- """A widget that expands the committed value beyond the fallback's own fill counts as a commit."""
- frame = _mock_frame(locator_count=1)
- skyvern_el = _mock_skyvern_element(frame)
- skyvern_el.get_attr = AsyncMock(return_value=None)
- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="I do not wish to disclose (private)")
- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)
-
- llm_response = {
- "auto_completion_attempt": True,
- "relevance_float": 0.9,
- "id": "OPT3",
- "value": "I do not wish to disclose",
- "reasoning": "Best semantic match",
- }
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
- ),
- patch(
- "skyvern.webeye.actions.handler.IncrementalScrapePage",
- return_value=inc_scrape,
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await discover_and_select_from_full_dropdown(
- context=_make_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- original_text="Decline to Self Identify",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
- assert isinstance(result, ActionSuccess)
-
-
-async def test_discover_fallback_exact_echo_without_expanded_state_is_a_commit() -> None:
- """A widget that commits the option text verbatim and exposes no expanded state is a commit, not a failure."""
- frame = _mock_frame(locator_count=1)
- skyvern_el = _mock_skyvern_element(frame)
- # No aria-expanded at all: the widget gives no open/closed signal, and the committed value is the
- # option text verbatim, so it is indistinguishable from the fallback's own fill by value alone.
- skyvern_el.get_attr = AsyncMock(return_value=None)
- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="I do not wish to disclose")
- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)
-
- llm_response = {
- "auto_completion_attempt": True,
- "relevance_float": 0.9,
- "id": "OPT3",
- "value": "I do not wish to disclose",
- "reasoning": "Best semantic match",
- }
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
- ),
- patch(
- "skyvern.webeye.actions.handler.IncrementalScrapePage",
- return_value=inc_scrape,
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await discover_and_select_from_full_dropdown(
- context=_make_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- original_text="Decline to Self Identify",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
- assert isinstance(result, ActionSuccess)
-
-
-async def test_discover_fallback_ignores_stale_cached_expanded_state() -> None:
- """A no-op Enter must not read as committed just because the element was scraped before the
- dropdown opened. get_attr is static-first, so the cached aria-expanded="false" from that scrape
- would otherwise stand in as commit evidence for an unchanged value."""
- frame = _mock_frame(locator_count=1)
- skyvern_el = _mock_skyvern_element(frame)
- # Behaves like the real SkyvernElement.get_attr: static cache says "false" (scraped while closed),
- # the live DOM says "true" (the listbox is still open, so Enter did nothing).
- static_attrs = {"aria-expanded": "false"}
- live_attrs = {"aria-expanded": "true"}
-
- async def _get_attr(name: str, mode: str = "auto", **_kwargs: object) -> object:
- if mode == "dynamic":
- return live_attrs.get(name)
- cached = static_attrs.get(name)
- return cached if cached is not None or mode == "static" else live_attrs.get(name)
-
- skyvern_el.get_attr = AsyncMock(side_effect=_get_attr)
- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="I do not wish to disclose")
- inc_scrape = _mock_incremental_scrape(DROPDOWN_OPTIONS)
-
- llm_response = {
- "auto_completion_attempt": True,
- "relevance_float": 0.9,
- "id": "OPT3",
- "value": "I do not wish to disclose",
- "reasoning": "Best semantic match",
- }
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
- ),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- with pytest.raises(AutoCompletionCommitFailure):
- await discover_and_select_from_full_dropdown(
- context=_make_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- original_text="Decline to Self Identify",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
-
@pytest.mark.asyncio
async def test_discover_fallback_returns_none_when_no_options() -> None:
"""ArrowDown produces no incremental elements → returns None."""
@@ -871,7 +612,7 @@ async def test_input_or_auto_complete_skips_discover_for_search_bar() -> None:
mock_prompt.load_prompt.return_value = "mocked prompt"
mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
- # Search bars fall through to plain typing (None) instead of hard-failing,
+ # Note: search bars bail out early at L2989 (before potential_values),
# so discover_fallback is never reached. This test verifies that.
result = await input_or_auto_complete_input(
input_or_select_context=_make_context(is_search_bar=True),
@@ -884,225 +625,6 @@ async def test_input_or_auto_complete_skips_discover_for_search_bar() -> None:
task=_TASK,
)
+ # Search bar returns None early, never reaching discover fallback
assert result is None
mock_discover.assert_not_called()
-
-
-async def test_input_or_auto_complete_failure_includes_attempt_trail() -> None:
- """A location field with the commit-required flag on returns a typed failure with the attempt trail."""
- skyvern_el = _mock_skyvern_element()
- choose_result = AutoCompletionResult(
- action_result=ActionFailure(AutoCompletionCommitFailure(stage="suggestions_never_rendered")),
- stage_outcome="suggestions_never_rendered",
- )
-
- with (
- patch(
- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",
- new=AsyncMock(return_value=choose_result),
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch(
- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
- new=AsyncMock(return_value=None),
- ),
- ):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await input_or_auto_complete_input(
- input_or_select_context=_make_context(is_location_input=True),
- scraped_page=MagicMock(),
- page=MagicMock(),
- dom=MagicMock(),
- text="private value",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
- assert isinstance(result, ActionFailure)
- assert result.exception_type == AutoCompletionCommitFailure.__name__
- assert result.exception_message is not None
- assert "attempt_1:suggestions_never_rendered" in result.exception_message
- assert "attempt_2:suggestion_not_matched" in result.exception_message
- assert "private value" not in result.exception_message
-
-
-def _side_effect_choose_result() -> AutoCompletionResult:
- return AutoCompletionResult(
- action_result=ActionFailure(AutoCompletionCommitFailure(stage="clicked_but_not_committed")),
- stage_outcome="clicked_but_not_committed",
- committed_side_effects=True,
- )
-
-
-async def test_side_effectful_click_stops_retrying_with_flag_on() -> None:
- """Once a click has mutated the field but could not be verified, the orchestration loop must
- stop instead of generating more values and clicking additional options on the dirty field."""
- skyvern_el = _mock_skyvern_element()
- choose_mock = AsyncMock(return_value=_side_effect_choose_result())
- discover_mock = AsyncMock(return_value=ActionSuccess())
-
- with (
- patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=choose_mock),
- patch("skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown", new=discover_mock),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await input_or_auto_complete_input(
- input_or_select_context=_make_context(is_location_input=True),
- scraped_page=MagicMock(),
- page=MagicMock(),
- dom=MagicMock(),
- text="private value",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
- assert choose_mock.call_count == 1
- discover_mock.assert_not_called()
- mock_app.SECONDARY_LLM_API_HANDLER.assert_not_called()
- assert isinstance(result, ActionFailure)
- assert result.exception_type == AutoCompletionCommitFailure.__name__
- assert "clicked_but_not_committed" in (result.exception_message or "")
-
-
-async def test_side_effectful_click_stops_retrying_with_flag_off() -> None:
- """With the flag off the loop still stops after a side-effectful click (returning None to fall
- through to plain typing) rather than clicking more options on the mutated field."""
- skyvern_el = _mock_skyvern_element()
- choose_mock = AsyncMock(return_value=_side_effect_choose_result())
- discover_mock = AsyncMock(return_value=ActionSuccess())
-
- with (
- patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=choose_mock),
- patch("skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown", new=discover_mock),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await input_or_auto_complete_input(
- input_or_select_context=_make_context(is_location_input=True),
- scraped_page=MagicMock(),
- page=MagicMock(),
- dom=MagicMock(),
- text="private value",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
- assert choose_mock.call_count == 1
- discover_mock.assert_not_called()
- assert result is None
-
-
-async def test_input_or_auto_complete_falls_through_when_flag_disabled() -> None:
- """With the commit-required flag off, an exhausted location autocomplete falls through to plain typing."""
- skyvern_el = _mock_skyvern_element()
- choose_result = AutoCompletionResult(
- action_result=ActionFailure(AutoCompletionCommitFailure(stage="suggestions_never_rendered")),
- stage_outcome="suggestions_never_rendered",
- )
-
- with (
- patch(
- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",
- new=AsyncMock(return_value=choose_result),
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch(
- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
- new=AsyncMock(return_value=None),
- ),
- ):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await input_or_auto_complete_input(
- input_or_select_context=_make_context(is_location_input=True),
- scraped_page=MagicMock(),
- page=MagicMock(),
- dom=MagicMock(),
- text="private value",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
- assert result is None
-
-
-async def test_fallback_stage_is_read_structurally_not_from_message_text() -> None:
- """The fallback stage comes from the exception's stage attribute, even when the
- message's attempt trail embeds every other stage name ahead of it."""
- skyvern_el = _mock_skyvern_element()
- choose_result = AutoCompletionResult(
- action_result=ActionFailure(AutoCompletionCommitFailure(stage="suggestions_never_rendered")),
- stage_outcome="suggestions_never_rendered",
- )
- fallback_failure = AutoCompletionCommitFailure(
- stage="clicked_but_not_committed",
- attempt_trail=["attempt_1:suggestions_never_rendered", "attempt_2:suggestion_not_matched"],
- )
- assert "suggestions_never_rendered" in str(fallback_failure)
-
- with (
- patch(
- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",
- new=AsyncMock(return_value=choose_result),
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch(
- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
- new=AsyncMock(side_effect=fallback_failure),
- ),
- ):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await input_or_auto_complete_input(
- input_or_select_context=_make_context(is_location_input=True),
- scraped_page=MagicMock(),
- page=MagicMock(),
- dom=MagicMock(),
- text="private value",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
-
- assert isinstance(result, ActionFailure)
- assert result.exception_type == AutoCompletionCommitFailure.__name__
- assert "attempt_2:clicked_but_not_committed" in (result.exception_message or "")
- assert "stage=clicked_but_not_committed" in (result.exception_message or "")
diff --git a/tests/unit/test_auto_completion_location.py b/tests/unit/test_auto_completion_location.py
index 778594980..f33020687 100644
--- a/tests/unit/test_auto_completion_location.py
+++ b/tests/unit/test_auto_completion_location.py
@@ -6,27 +6,23 @@ suggestion appears, we skip the LLM call and click the suggestion directly.
from __future__ import annotations
-import asyncio
import copy
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-import skyvern.webeye.actions.handler as handler_module
from skyvern.constants import SKYVERN_ID_ATTR
-from skyvern.exceptions import AutoCompletionCommitFailure, MissingElement, NoIncrementalElementFoundForAutoCompletion
+from skyvern.exceptions import NoIncrementalElementFoundForAutoCompletion
from skyvern.forge.sdk.models import StepStatus
from skyvern.webeye.actions.actions import InputOrSelectContext
from skyvern.webeye.actions.handler import (
AutoCompletionResult,
- _poll_autocomplete_incremental_elements,
_reset_autocomplete_for_llm_fallback,
- _verify_autocomplete_input_readback,
choose_auto_completion_dropdown,
input_or_auto_complete_input,
)
-from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
+from skyvern.webeye.actions.responses import ActionSuccess
from tests.unit.helpers import make_organization, make_step, make_task
# ---------------------------------------------------------------------------
@@ -38,15 +34,11 @@ _ORG = make_organization(_NOW)
_TASK = make_task(_NOW, _ORG, navigation_payload={"address": "123 Main St"})
_STEP = make_step(_NOW, _TASK, step_id="stp-1", status=StepStatus.created, order=0, output=None)
-SINGLE_ELEMENT = [
- {"id": "AAAA", "tagName": "li", "attributes": {"role": "option"}, "text": "123 Main St, Springfield, IL"}
-]
+SINGLE_ELEMENT = [{"id": "AAAA", "tag": "div", "text": "123 Main St, Springfield, IL"}]
MULTI_ELEMENTS = [
- {"id": "AAAA", "tagName": "li", "attributes": {"role": "option"}, "text": "123 Main St, Springfield, IL"},
- {"id": "AAAB", "tagName": "li", "attributes": {"role": "option"}, "text": "123 Main St, Springfield, MO"},
+ {"id": "AAAA", "tag": "div", "text": "123 Main St, Springfield, IL"},
+ {"id": "AAAB", "tag": "div", "text": "123 Main St, Springfield, MO"},
]
-LOADING_ELEMENT = {"id": "LOAD", "tagName": "div", "text": "Loading..."}
-OPTION_ELEMENT = {"id": "OPT", "tagName": "li", "attributes": {"role": "option"}, "text": "123 Main St, Springfield"}
def _make_location_context(**overrides: object) -> InputOrSelectContext:
@@ -76,767 +68,79 @@ def _mock_skyvern_element(frame: MagicMock | None = None) -> MagicMock:
el.get_frame.return_value = frame or _mock_frame()
el.get_frame_id.return_value = "frame-1"
el.is_interactable.return_value = True
- el.press_fill = AsyncMock()
- el.input_clear = AsyncMock()
- el.is_visible = AsyncMock(return_value=True)
- el.get_element_handler = AsyncMock(return_value=MagicMock())
- el.get_tag_name.return_value = "input"
- el.press_key = AsyncMock()
- el.get_attr = AsyncMock(return_value=None)
- input_locator = MagicMock()
- input_locator.input_value = AsyncMock(return_value="123 Main St, Springfield, IL")
- el.get_locator.return_value = input_locator
- return el
-
-
-def _mock_frame(locator_count: int = 1) -> MagicMock:
- """Return a mock Playwright Frame with a configurable locator."""
- frame = MagicMock()
- locator = MagicMock()
- locator.count = AsyncMock(return_value=locator_count)
- locator.click = AsyncMock()
- locator.element_handle = AsyncMock(return_value=MagicMock())
- locator.is_visible = AsyncMock(return_value=True)
- locator.get_attribute = AsyncMock(return_value=None)
- frame.locator.return_value = locator
- return frame
-
-
-def _mock_incremental_scrape(elements: list[dict]) -> MagicMock:
- """Return a mock IncrementalScrapePage that yields *elements*."""
- inc = MagicMock()
- inc.start_listen_dom_increment = AsyncMock()
- inc.stop_listen_dom_increment = AsyncMock()
- inc.get_incremental_elements_num = AsyncMock(return_value=len(elements))
- inc.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(elements))
- inc.build_html_tree.return_value = "<div>mocked</div>"
- return inc
-
-
-class _FakeSuggestionList:
- """Stateful listbox fake: each patched sleep advances the rendered frame."""
-
- def __init__(self, frames: list[list[dict]]) -> None:
- self._frames = frames
- self._tick = 0
-
- def _current(self) -> list[dict]:
- return self._frames[min(self._tick, len(self._frames) - 1)]
-
- async def advance(self, _delay: float) -> None:
- self._tick += 1
-
- def scrape(self) -> MagicMock:
- inc = MagicMock()
- inc.start_listen_dom_increment = AsyncMock()
- inc.stop_listen_dom_increment = AsyncMock()
- inc.get_incremental_elements_num = AsyncMock(side_effect=lambda: len(self._current()))
- inc.get_incremental_element_tree = AsyncMock(side_effect=lambda _cleanup: copy.deepcopy(self._current()))
- inc.build_html_tree.return_value = "<div>mocked</div>"
- return inc
-
-
-def test_auto_completion_result_does_not_default_to_success() -> None:
- assert AutoCompletionResult().action_result is None
-
-
-def test_removed_autocomplete_exception_name_still_catchable() -> None:
- """The deprecated OSS alias must keep downstream `except` clauses working for one release."""
- with pytest.raises(NoIncrementalElementFoundForAutoCompletion):
- raise AutoCompletionCommitFailure(stage="suggestion_not_matched")
-
-
- ("actual_value", "matched_label", "typed_prefix", "expected"),
- [
- ("123 MAIN ST., Springfield!", "123 Main St Springfield", "123 Main", True),
- ("Selected: 123 Main St, Springfield", "123 Main St Springfield", "123 Main", True),
- ("123 Main", "123 Main St Springfield", "123 Main", False),
- ("scala 5", "ca", "sc", False),
- ("123 Main St Springfield", "123 Main St, Springfield, USA", "123 Main", True),
- ],
-)
-async def test_autocomplete_readback_matches_on_token_boundaries(
- actual_value: str,
- matched_label: str,
- typed_prefix: str,
- expected: bool,
-) -> None:
- skyvern_el = _mock_skyvern_element()
- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value=actual_value)
-
- outcome = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_el,
- matched_label=matched_label,
- typed_prefix=typed_prefix,
- )
- assert outcome.committed is expected
-
-
-async def test_autocomplete_readback_rejects_lone_token_of_multiword_label() -> None:
- # A single token of a multi-word label (here "Springfield" of "123 Main St Springfield") no longer
- # identifies the selection, even though the value changed - the field must still cover a substantial
- # share of the label. Substantial truncation ("123 Main St" of the same label) stays a commit.
- lone_token = _mock_skyvern_element()
- lone_token.get_locator.return_value.input_value = AsyncMock(return_value="Springfield")
- weak = await _verify_autocomplete_input_readback(
- skyvern_element=lone_token,
- matched_label="123 Main St Springfield",
- typed_prefix="123 Main",
- )
- assert weak.committed is False
-
- substantial = _mock_skyvern_element()
- substantial.get_locator.return_value.input_value = AsyncMock(return_value="123 Main St")
- strong = await _verify_autocomplete_input_readback(
- skyvern_element=substantial,
- matched_label="123 Main St Springfield",
- typed_prefix="123",
- )
- assert strong.committed is True
-
-
-async def test_autocomplete_readback_requires_secondary_signal_when_label_equals_typed_text() -> None:
- skyvern_el = _mock_skyvern_element()
- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="Oakland, California")
-
- no_signal = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_el,
- matched_label="Oakland, California",
- typed_prefix="Oakland, California",
- )
- assert no_signal.committed is False
- assert no_signal.side_effects_observed is False
-
- list_closed = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_el,
- matched_label="Oakland, California",
- typed_prefix="Oakland, California",
- suggestions_closed=True,
- )
- assert list_closed.committed is True
-
- aria_selected = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_el,
- matched_label="Oakland, California",
- typed_prefix="Oakland, California",
- selection_state_present=True,
- )
- assert aria_selected.committed is True
-
-
-async def test_autocomplete_readback_accepts_cleared_input_only_when_list_closed() -> None:
- skyvern_el = _mock_skyvern_element()
- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="")
-
- chip_commit = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_el,
- matched_label="Oakland, California",
- typed_prefix="Oakland",
- suggestions_closed=True,
- )
- assert chip_commit.committed is True
-
- still_open = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_el,
- matched_label="Oakland, California",
- typed_prefix="Oakland",
- )
- assert still_open.committed is False
-
-
-async def test_autocomplete_readback_empty_value_needs_selection_signal_when_commit_required() -> None:
- # A click that dismisses the suggestions while clearing the field reads back identical to a chip
- # commit; under commit-required only an affirmative selection signal separates the two.
- skyvern_el = _mock_skyvern_element()
- skyvern_el.get_locator.return_value.input_value = AsyncMock(return_value="")
-
- dismissal = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_el,
- matched_label="Oakland, California",
- typed_prefix="Oakland",
- suggestions_closed=True,
- commit_required=True,
- )
- assert dismissal.committed is False
-
- affirmed = await _verify_autocomplete_input_readback(
- skyvern_element=skyvern_el,
- matched_label="Oakland, California",
- typed_prefix="Oakland",
- suggestions_closed=True,
- selection_state_present=True,
- commit_required=True,
- )
- assert affirmed.committed is True
-
-
-async def test_autocomplete_poll_returns_immediately_when_suggestions_exist() -> None:
- fake_list = _FakeSuggestionList([SINGLE_ELEMENT])
- incremental_scrape = fake_list.scrape()
-
- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)) as sleep:
- result = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scrape,
- cleanup_factory=MagicMock(),
- )
-
- assert result == SINGLE_ELEMENT
- sleep.assert_not_awaited()
-
-
-async def test_autocomplete_poll_waits_until_suggestions_render() -> None:
- fake_list = _FakeSuggestionList([[], [], SINGLE_ELEMENT])
- incremental_scrape = fake_list.scrape()
-
- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)) as sleep:
- result = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scrape,
- cleanup_factory=MagicMock(),
- )
-
- assert result == SINGLE_ELEMENT
- assert sleep.await_count == 2
-
-
-async def test_autocomplete_poll_keeps_waiting_past_loading_nodes_until_options_render() -> None:
- fake_list = _FakeSuggestionList([[LOADING_ELEMENT], [LOADING_ELEMENT], [LOADING_ELEMENT, OPTION_ELEMENT]])
- incremental_scrape = fake_list.scrape()
-
- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)):
- result = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scrape,
- cleanup_factory=MagicMock(),
- )
-
- assert OPTION_ELEMENT in result
-
-
-async def test_autocomplete_poll_reextracts_when_placeholder_replaced_in_place() -> None:
- # A listbox can swap a loading placeholder for an option in the same node, so the incremental
- # count stays constant (1 -> 1). A count-delta guard would extract the placeholder once and never
- # re-extract, missing the rendered option.
- fake_list = _FakeSuggestionList([[LOADING_ELEMENT], [OPTION_ELEMENT]])
- incremental_scrape = fake_list.scrape()
-
- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)):
- result = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scrape,
- cleanup_factory=MagicMock(),
- )
-
- assert OPTION_ELEMENT in result
-
-
-async def test_autocomplete_poll_is_bounded_when_suggestions_never_render() -> None:
- fake_list = _FakeSuggestionList([[]])
- incremental_scrape = fake_list.scrape()
-
- with patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock(side_effect=fake_list.advance)) as sleep:
- result = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scrape,
- cleanup_factory=MagicMock(),
- )
-
- assert result == []
- incremental_scrape.get_incremental_element_tree.assert_not_awaited()
- assert sleep.await_count <= handler_module.AUTOCOMPLETE_POLL_MAX_ATTEMPTS
-
-
-async def test_autocomplete_poll_does_not_cancel_slow_extraction(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(handler_module, "AUTOCOMPLETE_POLL_INTERVAL_SECONDS", 0.05)
- incremental_scrape = MagicMock()
- incremental_scrape.get_incremental_elements_num = AsyncMock(return_value=1)
-
- async def _slow_extract(_cleanup: object) -> list[dict]:
- await asyncio.sleep(0.2)
- return copy.deepcopy(SINGLE_ELEMENT)
-
- incremental_scrape.get_incremental_element_tree = AsyncMock(side_effect=_slow_extract)
-
- result = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scrape,
- cleanup_factory=MagicMock(),
- )
-
- assert result == SINGLE_ELEMENT
- assert incremental_scrape.get_incremental_element_tree.await_count == 1
-
-
-async def test_autocomplete_poll_bounds_a_hanging_probe(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(handler_module, "AUTOCOMPLETE_POLL_INTERVAL_SECONDS", 0.02)
- monkeypatch.setattr(handler_module, "AUTOCOMPLETE_POLL_MAX_ATTEMPTS", 2)
- incremental_scrape = MagicMock()
-
- async def _hanging_probe() -> int:
- await asyncio.sleep(10)
- return 1
-
- incremental_scrape.get_incremental_elements_num = AsyncMock(side_effect=_hanging_probe)
- incremental_scrape.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(SINGLE_ELEMENT))
-
- result = await _poll_autocomplete_incremental_elements(
- incremental_scraped=incremental_scrape,
- cleanup_factory=MagicMock(),
- )
-
- assert result == []
- incremental_scrape.get_incremental_element_tree.assert_not_awaited()
-
-
-async def test_autocomplete_timeout_reports_suggestions_never_rendered() -> None:
- skyvern_el = _mock_skyvern_element()
- incremental_scrape = _mock_incremental_scrape([])
- scraped_page = MagicMock(id_to_css_dict={})
- scraped_after_open = MagicMock(id_to_css_dict={})
- scraped_page.generate_scraped_page_without_screenshots = AsyncMock(return_value=scraped_after_open)
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock()),
- ),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=incremental_scrape),
- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),
- ):
- result = await choose_auto_completion_dropdown(
- context=_make_location_context(),
- page=MagicMock(),
- scraped_page=scraped_page,
- dom=MagicMock(),
- text="123 Main",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- is_location_input=True,
- )
-
- assert isinstance(result.action_result, ActionFailure)
- assert result.action_result.exception_type == AutoCompletionCommitFailure.__name__
- assert "suggestions_never_rendered" in (result.action_result.exception_message or "")
-
-
-# ---------------------------------------------------------------------------
-# Tests for choose_auto_completion_dropdown
-# ---------------------------------------------------------------------------
-
-
-async def test_location_single_option_skips_llm() -> None:
- """When is_location_input=True and exactly 1 option appears, the LLM must NOT be called."""
- frame = _mock_frame(locator_count=1)
- skyvern_el = _mock_skyvern_element(frame)
- inc_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
- ),
- patch(
- "skyvern.webeye.actions.handler.IncrementalScrapePage",
- return_value=inc_scrape,
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()
-
- result = await choose_auto_completion_dropdown(
- context=_make_location_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="123 Main St",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- is_location_input=True,
- )
-
- # The LLM should never have been called
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()
-
- # The locator should have been clicked
- frame.locator.assert_called_with(f'[{SKYVERN_ID_ATTR}="AAAA"]')
- frame.locator.return_value.click.assert_awaited_once()
-
- # Result should indicate success
- assert isinstance(result.action_result, ActionSuccess)
-
-
-async def test_location_single_option_click_with_side_effects_does_not_refire() -> None:
- """A click that changed the value (but did not match) must hard-fail instead of re-firing."""
- frame, option_locator, skyvern_el, skyvern_frame = _mock_autocomplete_input(
- selected_value="Wrong city",
- click_closes_listbox=False,
- )
- inc_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=skyvern_frame),
- ),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()
-
- result = await choose_auto_completion_dropdown(
- context=_make_location_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="123 Main",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- is_location_input=True,
- )
-
- option_locator.click.assert_awaited_once()
- skyvern_el.press_fill.assert_awaited_once()
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()
- assert isinstance(result.action_result, ActionFailure)
- assert result.action_result.exception_type == AutoCompletionCommitFailure.__name__
- assert "clicked_but_not_committed" in (result.action_result.exception_message or "")
-
-
-async def test_location_single_option_side_effect_free_noncommit_falls_back_to_llm() -> None:
- """A no-op click (value unchanged, listbox still open) resets and runs the LLM chooser in the same call."""
- _frame, option_locator, skyvern_el, skyvern_frame = _mock_autocomplete_input(
- selected_value="123 Main",
- click_closes_listbox=False,
- )
- events: list[str] = []
- skyvern_el.input_clear = AsyncMock(side_effect=lambda: events.append("clear"))
- skyvern_el.press_fill = AsyncMock(side_effect=lambda value: events.append(f"fill:{value}"))
- skyvern_el.press_key = AsyncMock()
- initial_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)
- initial_scrape.build_html_tree.return_value = "<div>stale options</div>"
- fallback_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)
- fallback_scrape.build_html_tree.return_value = "<div>fresh options</div>"
-
- llm_response = {
- "auto_completion_attempt": False,
- "relevance_float": 0.0,
- "id": "",
- "direct_searching": True,
- "value": "123 Main",
- "reasoning": "Read-back mismatch",
- }
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=skyvern_frame),
- ),
- patch(
- "skyvern.webeye.actions.handler.IncrementalScrapePage",
- side_effect=[initial_scrape, fallback_scrape],
- ) as scrape_factory,
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
-
- async def _llm_handler(**_: object) -> dict[str, object]:
- events.append("llm")
- return llm_response
-
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(side_effect=_llm_handler)
- mock_app.AGENT_FUNCTION = MagicMock()
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await choose_auto_completion_dropdown(
- context=_make_location_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="123 Main",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- is_location_input=True,
- )
-
- option_locator.click.assert_awaited_once()
- assert scrape_factory.call_count == 2
- skyvern_el.input_clear.assert_awaited_once()
- assert skyvern_el.press_fill.await_count == 2
- assert events == ["fill:123 Main", "clear", "fill:123 Main", "llm"]
- assert mock_prompt.load_prompt.call_args.kwargs["elements"] == "<div>fresh options</div>"
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_awaited_once()
- skyvern_el.press_key.assert_awaited_once_with("Enter")
- assert isinstance(result.action_result, ActionSuccess)
-
-
-async def test_direct_searching_unchanged_value_fails_when_commit_required() -> None:
- """Searching with the typed text leaves the value unchanged, which is indistinguishable from Enter
- doing nothing. On a field that must commit a real suggestion that is a failure, not a success."""
- _frame, _option_locator, skyvern_el, skyvern_frame = _mock_autocomplete_input(
- selected_value="123 Main",
- click_closes_listbox=False,
- )
- skyvern_el.input_clear = AsyncMock()
- skyvern_el.press_fill = AsyncMock()
- skyvern_el.press_key = AsyncMock()
- initial_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)
- initial_scrape.build_html_tree.return_value = "<div>stale options</div>"
- fallback_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)
- fallback_scrape.build_html_tree.return_value = "<div>fresh options</div>"
-
- llm_response = {
- "auto_completion_attempt": False,
- "relevance_float": 0.0,
- "id": "",
- "direct_searching": True,
- "value": "123 Main",
- "reasoning": "Read-back mismatch",
- }
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=skyvern_frame),
- ),
- patch(
- "skyvern.webeye.actions.handler.IncrementalScrapePage",
- side_effect=[initial_scrape, fallback_scrape],
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
- mock_app.AGENT_FUNCTION = MagicMock()
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- result = await choose_auto_completion_dropdown(
- context=_make_location_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="123 Main",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- is_location_input=True,
- commit_required=True,
- )
-
- assert isinstance(result.action_result, ActionFailure)
- assert result.action_result.exception_type == AutoCompletionCommitFailure.__name__
- assert "clicked_but_not_committed" in (result.action_result.exception_message or "")
-
-
-async def test_unexpected_exception_surfaces_as_itself_not_commit_failure() -> None:
- """A genuine bug-shaped error keeps its identity in the ActionFailure instead of being
- rewrapped as an expected no-match commit failure."""
- skyvern_el = _mock_skyvern_element()
- skyvern_el.press_fill = AsyncMock(side_effect=MissingElement(element_id="AAAA"))
- inc_scrape = _mock_incremental_scrape([])
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock()),
- ),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),
- ):
- result = await choose_auto_completion_dropdown(
- context=_make_location_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="123 Main",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- is_location_input=True,
- )
-
- assert isinstance(result.action_result, ActionFailure)
- assert result.action_result.exception_type == MissingElement.__name__
- assert result.stage_outcome == "suggestion_not_matched"
-
-
-async def test_commit_failure_keeps_its_stage_in_action_failure() -> None:
- """An AutoCompletionCommitFailure raised inside the dropdown flow keeps its own stage."""
- skyvern_el = _mock_skyvern_element()
- skyvern_el.press_fill = AsyncMock(side_effect=AutoCompletionCommitFailure(stage="suggestions_never_rendered"))
- inc_scrape = _mock_incremental_scrape([])
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock()),
- ),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),
- ):
- result = await choose_auto_completion_dropdown(
- context=_make_location_context(),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="123 Main",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- is_location_input=True,
- )
+ el.press_fill = AsyncMock()
+ el.input_clear = AsyncMock()
+ el.is_visible = AsyncMock(return_value=True)
+ el.get_element_handler = AsyncMock(return_value=MagicMock())
+ return el
- assert isinstance(result.action_result, ActionFailure)
- assert result.action_result.exception_type == AutoCompletionCommitFailure.__name__
- assert result.stage_outcome == "suggestions_never_rendered"
+def _mock_frame(locator_count: int = 1) -> MagicMock:
+ """Return a mock Playwright Frame with a configurable locator."""
+ frame = MagicMock()
+ locator = MagicMock()
+ locator.count = AsyncMock(return_value=locator_count)
+ locator.click = AsyncMock()
+ frame.locator.return_value = locator
+ return frame
-async def test_unexpected_exception_still_hard_fails_location_field_when_flag_on() -> None:
- """A preserved unexpected exception from the dropdown flow still produces the staged
- location failure when the commit-required flag is on."""
- skyvern_el = _mock_skyvern_element()
- choose_result = AutoCompletionResult(
- action_result=ActionFailure(MissingElement(element_id="AAAA")),
- stage_outcome="suggestion_not_matched",
- )
- with (
- patch(
- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",
- new=AsyncMock(return_value=choose_result),
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch(
- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
- new=AsyncMock(return_value=None),
- ),
- ):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
+def _mock_incremental_scrape(elements: list[dict]) -> MagicMock:
+ """Return a mock IncrementalScrapePage that yields *elements*."""
+ inc = MagicMock()
+ inc.start_listen_dom_increment = AsyncMock()
+ inc.stop_listen_dom_increment = AsyncMock()
+ inc.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(elements))
+ inc.build_html_tree.return_value = "<div>mocked</div>"
+ return inc
- result = await input_or_auto_complete_input(
- input_or_select_context=_make_location_context(),
- scraped_page=MagicMock(),
- page=MagicMock(),
- dom=MagicMock(),
- text="123 Main St",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
- assert isinstance(result, ActionFailure)
- assert result.exception_type == AutoCompletionCommitFailure.__name__
- assert "attempt_1:suggestion_not_matched" in (result.exception_message or "")
+# ---------------------------------------------------------------------------
+# Tests for choose_auto_completion_dropdown
+# ---------------------------------------------------------------------------
@pytest.mark.asyncio
-async def test_location_search_bar_keeps_fallthrough_even_with_flag_on() -> None:
- """A location-search widget is both a location input and a search bar. Search bars keep their
- documented fall-through, so strict commit mode must not claim one and hard-fail it."""
- skyvern_el = _mock_skyvern_element()
- choose_result = AutoCompletionResult(
- action_result=ActionFailure(MissingElement(element_id="AAAA")),
- stage_outcome="suggestion_not_matched",
- )
+async def test_location_single_option_skips_llm() -> None:
+ """When is_location_input=True and exactly 1 option appears, the LLM must NOT be called."""
+ frame = _mock_frame(locator_count=1)
+ skyvern_el = _mock_skyvern_element(frame)
+ inc_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)
with (
patch(
- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",
- new=AsyncMock(return_value=choose_result),
+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
+ new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
patch(
- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
- new=AsyncMock(return_value=None),
+ "skyvern.webeye.actions.handler.IncrementalScrapePage",
+ return_value=inc_scrape,
),
+ patch("skyvern.webeye.actions.handler.app") as mock_app,
):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=True)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()
- result = await input_or_auto_complete_input(
- input_or_select_context=_make_location_context(is_search_bar=True),
- scraped_page=MagicMock(),
+ result = await choose_auto_completion_dropdown(
+ context=_make_location_context(),
page=MagicMock(),
+ scraped_page=MagicMock(),
dom=MagicMock(),
text="123 Main St",
skyvern_element=skyvern_el,
step=_STEP,
task=_TASK,
+ is_location_input=True,
)
- assert not isinstance(result, ActionFailure)
-
-
-async def test_unexpected_exception_falls_through_to_plain_typing_when_flag_off() -> None:
- """With the commit-required flag off, a preserved unexpected exception still lets the
- location field fall through to plain typing."""
- skyvern_el = _mock_skyvern_element()
- choose_result = AutoCompletionResult(
- action_result=ActionFailure(MissingElement(element_id="AAAA")),
- stage_outcome="suggestion_not_matched",
- )
-
- with (
- patch(
- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",
- new=AsyncMock(return_value=choose_result),
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch(
- "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
- new=AsyncMock(return_value=None),
- ),
- ):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=False)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
+ # The LLM should never have been called
+ mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()
- result = await input_or_auto_complete_input(
- input_or_select_context=_make_location_context(),
- scraped_page=MagicMock(),
- page=MagicMock(),
- dom=MagicMock(),
- text="123 Main St",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- )
+ # The locator should have been clicked
+ frame.locator.assert_called_with(f'[{SKYVERN_ID_ATTR}="AAAA"]')
+ frame.locator.return_value.click.assert_awaited_once()
- assert result is None
+ # Result should indicate success
+ assert isinstance(result.action_result, ActionSuccess)
@pytest.mark.asyncio
@@ -891,7 +195,6 @@ async def test_location_multiple_options_calls_llm() -> None:
"direct_searching": False,
"reasoning": "First option matches",
}
- selected_element = MagicMock(scroll_into_view=AsyncMock(), click=AsyncMock())
with (
patch(
@@ -905,14 +208,13 @@ async def test_location_multiple_options_calls_llm() -> None:
patch("skyvern.webeye.actions.handler.app") as mock_app,
patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch("skyvern.webeye.actions.handler.SkyvernElement", return_value=selected_element),
):
mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
mock_app.AGENT_FUNCTION = MagicMock()
mock_prompt.load_prompt.return_value = "mocked prompt"
mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
- result = await choose_auto_completion_dropdown(
+ await choose_auto_completion_dropdown(
context=_make_location_context(),
page=MagicMock(),
scraped_page=MagicMock(),
@@ -926,8 +228,6 @@ async def test_location_multiple_options_calls_llm() -> None:
# LLM should have been called because there are 2 options
mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_awaited_once()
- selected_element.click.assert_awaited_once()
- assert isinstance(result.action_result, ActionSuccess)
@pytest.mark.asyncio
@@ -1120,44 +420,12 @@ async def test_input_or_auto_complete_passes_false_for_non_location() -> None:
assert call_kwargs["is_location_input"] is False
[email protected]("flag_enabled", [True, False])
-async def test_input_or_auto_complete_gates_verification_on_the_flag(flag_enabled: bool) -> None:
- """Commit verification runs only behind the flag; flag-off keeps the pre-verification behavior."""
- context = _make_location_context()
-
- with (
- patch(
- "skyvern.webeye.actions.handler.choose_auto_completion_dropdown",
- new=AsyncMock(return_value=AutoCompletionResult(action_result=ActionSuccess())),
- ) as mock_choose,
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- ):
- mock_app.EXPERIMENTATION_PROVIDER.is_feature_enabled_cached = AsyncMock(return_value=flag_enabled)
-
- result = await input_or_auto_complete_input(
- input_or_select_context=context,
- scraped_page=MagicMock(),
- page=MagicMock(),
- dom=MagicMock(),
- text="123 Main St",
- skyvern_element=_mock_skyvern_element(),
- step=_STEP,
- task=_TASK,
- )
-
- assert isinstance(result, ActionSuccess)
- call_kwargs = mock_choose.call_args.kwargs
- assert call_kwargs["verify_commit"] is flag_enabled
- assert call_kwargs["commit_required"] is flag_enabled
-
-
# ---------------------------------------------------------------------------
# Integration tests: options that don't contain the input fall through to LLM
# ---------------------------------------------------------------------------
-NO_RESULT_ELEMENTS = [{"id": "AAAA", "tagName": "div", "interactable": True, "text": "No results"}]
-UNRELATED_ELEMENTS = [{"id": "AAAA", "tagName": "div", "interactable": True, "text": "Something completely different"}]
+NO_RESULT_ELEMENTS = [{"id": "AAAA", "tag": "div", "text": "No results"}]
+UNRELATED_ELEMENTS = [{"id": "AAAA", "tag": "div", "text": "Something completely different"}]
DETERMINISTIC_ELEMENTS = [
{"id": "AAAA", "tagName": "li", "attributes": {"role": "option"}, "text": "San Francisco, California"},
{"id": "AAAB", "tagName": "li", "attributes": {"role": "option"}, "text": "Oakland, California"},
@@ -1172,20 +440,12 @@ def _mock_autocomplete_input(
*,
selected_value: str,
option_identity: dict[str, object] | None = None,
- click_closes_listbox: bool = True,
) -> tuple[MagicMock, MagicMock, MagicMock, MagicMock]:
frame = MagicMock()
option_locator = MagicMock()
option_locator.count = AsyncMock(return_value=1)
option_locator.element_handle = AsyncMock(return_value=MagicMock())
- option_locator.is_visible = AsyncMock(return_value=True)
- option_locator.get_attribute = AsyncMock(return_value=None)
-
- async def _click(**_: object) -> None:
- if click_closes_listbox:
- option_locator.count = AsyncMock(return_value=0)
-
- option_locator.click = AsyncMock(side_effect=_click)
+ option_locator.click = AsyncMock()
frame.locator.return_value = option_locator
input_locator = MagicMock()
@@ -1215,7 +475,6 @@ async def test_location_no_results_option_falls_through_to_llm() -> None:
"relevance_float": 0.0,
"id": "",
"direct_searching": True,
- "value": "123 Main St, Springfield, IL",
"reasoning": "No results shown",
}
@@ -1392,7 +651,6 @@ async def test_collapse_autocomplete_search_bar_exact_match_uses_llm() -> None:
"relevance_float": 0.0,
"id": "",
"direct_searching": True,
- "value": "Oakland, California",
"reasoning": "Search bars should direct search",
}
@@ -1411,7 +669,7 @@ async def test_collapse_autocomplete_search_bar_exact_match_uses_llm() -> None:
mock_prompt.load_prompt.return_value = "mocked prompt"
mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
- result = await choose_auto_completion_dropdown(
+ await choose_auto_completion_dropdown(
context=_make_non_location_context(field="Search", is_search_bar=True),
page=MagicMock(),
scraped_page=MagicMock(),
@@ -1426,7 +684,6 @@ async def test_collapse_autocomplete_search_bar_exact_match_uses_llm() -> None:
option_locator.click.assert_not_called()
mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_awaited_once()
skyvern_el.press_key.assert_awaited_once_with("Enter")
- assert isinstance(result.action_result, ActionSuccess)
@pytest.mark.asyncio
@@ -1443,7 +700,6 @@ async def test_collapse_autocomplete_ambiguous_exact_match_falls_back_to_llm() -
"relevance_float": 0.0,
"id": "",
"direct_searching": True,
- "value": "Oakland, California",
"reasoning": "Ambiguous duplicate labels",
}
@@ -1502,7 +758,6 @@ async def test_collapse_autocomplete_identity_mismatch_resets_before_llm() -> No
"relevance_float": 0.0,
"id": "",
"direct_searching": True,
- "value": "Oakland, California",
"reasoning": "Option rerendered",
}
@@ -1554,11 +809,10 @@ async def test_collapse_autocomplete_identity_mismatch_resets_before_llm() -> No
@pytest.mark.asyncio
async def test_collapse_autocomplete_readback_mismatch_falls_back_to_llm() -> None:
- """A no-op deterministic click (value unchanged, listbox open) is not accepted and resets before the LLM."""
+ """A deterministic click is not accepted when the input read-back is not the matched label."""
_frame, option_locator, skyvern_el, skyvern_frame_mock = _mock_autocomplete_input(
- selected_value="Oakland, California",
+ selected_value="Wrong city",
option_identity={"index": 1, "label": "Oakland, California"},
- click_closes_listbox=False,
)
events: list[str] = []
skyvern_el.input_clear = AsyncMock(side_effect=lambda: events.append("clear"))
@@ -1576,7 +830,6 @@ async def test_collapse_autocomplete_readback_mismatch_falls_back_to_llm() -> No
"relevance_float": 0.0,
"id": "",
"direct_searching": True,
- "value": "Oakland, California",
"reasoning": "Read-back mismatch",
}
@@ -1625,149 +878,6 @@ async def test_collapse_autocomplete_readback_mismatch_falls_back_to_llm() -> No
skyvern_el.press_key.assert_awaited_once_with("Enter")
-async def test_collapse_autocomplete_deterministic_path_verifies_even_when_flag_off() -> None:
- # The deterministic read-back is the one verify site that exists on main, so it stays on even at
- # flag-off: a click that leaves the value unchanged with the listbox open is not a commit and must
- # still reset and fall back to the LLM chooser, never be reported as a silent success.
- _frame, option_locator, skyvern_el, skyvern_frame_mock = _mock_autocomplete_input(
- selected_value="Oakland, California",
- option_identity={"index": 1, "label": "Oakland, California"},
- click_closes_listbox=False,
- )
- events: list[str] = []
- skyvern_el.input_clear = AsyncMock(side_effect=lambda: events.append("clear"))
- skyvern_el.press_fill = AsyncMock(side_effect=lambda value: events.append(f"fill:{value}"))
- skyvern_el.press_key = AsyncMock()
- initial_scrape = _mock_incremental_scrape(DETERMINISTIC_ELEMENTS)
- fallback_scrape = _mock_incremental_scrape(
- [{"id": "AAAB", "tagName": "li", "attributes": {"role": "option"}, "text": "Oakland, California"}]
- )
- llm_response = {
- "auto_completion_attempt": False,
- "relevance_float": 0.0,
- "id": "",
- "direct_searching": True,
- "value": "Oakland, California",
- "reasoning": "Read-back mismatch",
- }
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=skyvern_frame_mock),
- ),
- patch(
- "skyvern.webeye.actions.handler.IncrementalScrapePage",
- side_effect=[initial_scrape, fallback_scrape],
- ),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- ):
-
- async def _llm_handler(**_: object) -> dict[str, object]:
- events.append("llm")
- return llm_response
-
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(side_effect=_llm_handler)
- mock_app.AGENT_FUNCTION = MagicMock()
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
-
- await choose_auto_completion_dropdown(
- context=_make_non_location_context(field="Current location"),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="Oakland, California",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- verify_commit=False,
- collapse_autocomplete_fanout_enabled=True,
- )
-
- option_locator.click.assert_awaited_once()
- skyvern_el.get_locator.return_value.input_value.assert_awaited()
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_awaited_once()
-
-
-async def test_collapse_autocomplete_chip_widget_commit_succeeds() -> None:
- """A chip/token widget clears the input after committing; cleared value + closed listbox is a commit."""
- _frame, option_locator, skyvern_el, skyvern_frame_mock = _mock_autocomplete_input(
- selected_value="",
- option_identity={"index": 1, "label": "Oakland, California"},
- )
- inc_scrape = _mock_incremental_scrape(DETERMINISTIC_ELEMENTS)
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=skyvern_frame_mock),
- ),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()
-
- result = await choose_auto_completion_dropdown(
- context=_make_non_location_context(field="Current location"),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="Oakland, California",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- collapse_autocomplete_fanout_enabled=True,
- )
-
- option_locator.click.assert_awaited_once()
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()
- assert isinstance(result.action_result, ActionSuccess)
-
-
-async def test_collapse_autocomplete_click_with_side_effects_does_not_refire() -> None:
- """A click that changed the value (but did not match) must not be re-fired via the LLM fallback."""
- _frame, option_locator, skyvern_el, skyvern_frame_mock = _mock_autocomplete_input(
- selected_value="Wrong city",
- option_identity={"index": 1, "label": "Oakland, California"},
- click_closes_listbox=False,
- )
- inc_scrape = _mock_incremental_scrape(DETERMINISTIC_ELEMENTS)
-
- with (
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=skyvern_frame_mock),
- ),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- ):
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock()
-
- result = await choose_auto_completion_dropdown(
- context=_make_non_location_context(field="Current location"),
- page=MagicMock(),
- scraped_page=MagicMock(),
- dom=MagicMock(),
- text="Oakland, California",
- skyvern_element=skyvern_el,
- step=_STEP,
- task=_TASK,
- collapse_autocomplete_fanout_enabled=True,
- )
-
- option_locator.click.assert_awaited_once()
- skyvern_el.press_fill.assert_awaited_once()
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER.assert_not_called()
- assert isinstance(result.action_result, ActionFailure)
- assert "clicked_but_not_committed" in (result.action_result.exception_message or "")
-
-
@pytest.mark.asyncio
async def test_collapse_autocomplete_detached_option_resets_before_llm() -> None:
"""A deterministic candidate that detaches before the click must refresh before the LLM chooser."""
@@ -1792,7 +902,6 @@ async def test_collapse_autocomplete_detached_option_resets_before_llm() -> None
"relevance_float": 0.0,
"id": "",
"direct_searching": True,
- "value": "Oakland, California",
"reasoning": "Option detached",
}
@@ -1871,7 +980,6 @@ async def test_reset_autocomplete_empty_incremental_rescrapes_page_elements() ->
with (
patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=fallback_scrape) as scrape_factory,
patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_after_open),
- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),
):
(
returned_scrape,
@@ -1894,9 +1002,9 @@ async def test_reset_autocomplete_empty_incremental_rescrapes_page_elements() ->
current_scrape.stop_listen_dom_increment.assert_awaited_once()
skyvern_el.input_clear.assert_awaited_once()
skyvern_el.press_fill.assert_awaited_once_with("Oakland")
+ skyvern_frame.safe_wait_for_animation_end.assert_awaited_once()
scrape_factory.assert_called_once_with(skyvern_frame=skyvern_frame)
- fallback_scrape.get_incremental_elements_num.assert_awaited()
- fallback_scrape.get_incremental_element_tree.assert_not_awaited()
+ fallback_scrape.get_incremental_element_tree.assert_awaited_once()
scraped_page.generate_scraped_page_without_screenshots.assert_awaited_once()
dom_after_open.get_skyvern_element_by_id.assert_awaited_once_with("FRESH")
assert returned_scrape is fallback_scrape
@@ -1932,8 +1040,7 @@ async def test_reset_autocomplete_empty_rescrape_without_interactable_elements_r
with (
patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=fallback_scrape),
patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_after_open),
- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),
- pytest.raises(AutoCompletionCommitFailure),
+ pytest.raises(NoIncrementalElementFoundForAutoCompletion),
):
await _reset_autocomplete_for_llm_fallback(
current_incremental_scraped=current_scrape,
diff --git a/tests/unit/test_blocked_form_advance_followup.py b/tests/unit/test_blocked_form_advance_followup.py
deleted file mode 100644
index 8cee8faba..000000000
--- a/tests/unit/test_blocked_form_advance_followup.py
+++ /dev/null
@@ -1,134 +0,0 @@
-from types import SimpleNamespace
-from unittest.mock import AsyncMock, Mock
-
-import pytest
-
-from skyvern.webeye.actions import handler
-from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
-
-
-def _element(tag: str, *, attributes: dict | None = None, text: str | None = None) -> dict:
- return {"tagName": tag, "attributes": attributes or {}, "text": text}
-
-
- ("element", "expected"),
- [
- (_element("button", text="Next"), True),
- (_element("button", text="Continue"), True),
- (_element("button", text="Save and proceed"), True),
- (_element("input", attributes={"type": "submit", "value": "Go"}), True),
- (_element("input", attributes={"type": "image"}), True),
- (_element("a", attributes={"role": "button"}, text="Confirm"), True),
- (_element("button", attributes={"aria-label": "Submit application"}), True),
- (_element("button", text="Add another item"), False),
- (_element("input", attributes={"type": "checkbox"}), False),
- (_element("div", text="Next"), False),
- (_element("a", text="Next"), False),
- (_element("button", text=""), False),
- ],
-)
-def test_click_is_submit_like(element: dict, expected: bool) -> None:
- assert handler._click_is_submit_like(element) is expected
-
-
-def _submit_element() -> object:
- return SimpleNamespace(
- get_element_dict=Mock(return_value=_element("button", text="Next")),
- get_id=Mock(return_value="next-btn"),
- )
-
-
-async def test_blocked_form_advance_sets_followup_on_non_advancing_submit(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))
- results: list[object] = [ActionSuccess()]
- await handler._maybe_flag_blocked_form_advance(
- results=results,
- skyvern_element=_submit_element(),
- page=SimpleNamespace(url="https://example.com/step"),
- original_url="https://example.com/step",
- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),
- task=SimpleNamespace(organization_id=None),
- )
- assert results[-1].needs_followup is True
- assert results[-1].followup_message
- assert "Next" in results[-1].followup_message
-
-
-async def test_blocked_form_advance_noop_when_url_changed(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))
- results: list[object] = [ActionSuccess()]
- await handler._maybe_flag_blocked_form_advance(
- results=results,
- skyvern_element=_submit_element(),
- page=SimpleNamespace(url="https://example.com/next-step"),
- original_url="https://example.com/step",
- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),
- task=SimpleNamespace(organization_id=None),
- )
- assert results[-1].needs_followup is None
-
-
-async def test_blocked_form_advance_noop_when_new_elements_appeared(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))
- results: list[object] = [ActionSuccess()]
- await handler._maybe_flag_blocked_form_advance(
- results=results,
- skyvern_element=_submit_element(),
- page=SimpleNamespace(url="https://example.com/step"),
- original_url="https://example.com/step",
- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=5)),
- task=SimpleNamespace(organization_id=None),
- )
- assert results[-1].needs_followup is None
-
-
-async def test_blocked_form_advance_noop_when_not_submit_like(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))
- results: list[object] = [ActionSuccess()]
- await handler._maybe_flag_blocked_form_advance(
- results=results,
- skyvern_element=SimpleNamespace(
- get_element_dict=Mock(return_value=_element("input", attributes={"type": "checkbox"})),
- get_id=Mock(return_value="cb"),
- ),
- page=SimpleNamespace(url="https://example.com/step"),
- original_url="https://example.com/step",
- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),
- task=SimpleNamespace(organization_id=None),
- )
- assert results[-1].needs_followup is None
-
-
-async def test_blocked_form_advance_noop_when_flag_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=False))
- results: list[object] = [ActionSuccess()]
- await handler._maybe_flag_blocked_form_advance(
- results=results,
- skyvern_element=_submit_element(),
- page=SimpleNamespace(url="https://example.com/step"),
- original_url="https://example.com/step",
- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),
- task=SimpleNamespace(organization_id=None),
- )
- assert results[-1].needs_followup is None
-
-
-async def test_blocked_form_advance_noop_when_last_result_not_success(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(handler, "_is_surface_blocked_form_advance_enabled", AsyncMock(return_value=True))
- results: list[object] = [ActionFailure(Exception("boom"))]
- await handler._maybe_flag_blocked_form_advance(
- results=results,
- skyvern_element=_submit_element(),
- page=SimpleNamespace(url="https://example.com/step"),
- original_url="https://example.com/step",
- incremental_scraped=SimpleNamespace(get_incremental_elements_num=AsyncMock(return_value=0)),
- task=SimpleNamespace(organization_id=None),
- )
- assert results[-1].needs_followup is None
diff --git a/tests/unit/test_input_text_combobox_commit.py b/tests/unit/test_input_text_combobox_commit.py
index a8984d5ce..a927b8592 100644
--- a/tests/unit/test_input_text_combobox_commit.py
+++ b/tests/unit/test_input_text_combobox_commit.py
@@ -206,10 +206,6 @@ async def _run_combobox_input(
),
patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),
patch("skyvern.webeye.actions.handler.sequentially_select_from_dropdown", new=select_mock),
- patch(
- "skyvern.webeye.actions.handler._is_input_text_commit_verification_enabled",
- new=AsyncMock(return_value=False),
- ),
):
results = await handle_input_text_action(
action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP
diff --git a/tests/unit/test_input_text_masked_readback.py b/tests/unit/test_input_text_masked_readback.py
deleted file mode 100644
index 16313e34c..000000000
--- a/tests/unit/test_input_text_masked_readback.py
+++ /dev/null
@@ -1,190 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-from skyvern.exceptions import MaskedInputReadbackMismatch
-from skyvern.forge.sdk.models import StepStatus
-from skyvern.webeye.actions.actions import InputTextAction
-from skyvern.webeye.actions.handler import (
- _committed_value_is_masked_incomplete,
- _has_input_mask_evidence,
- _verify_masked_input_after_fill,
- handle_input_text_action,
-)
-from skyvern.webeye.actions.responses import ActionFailure
-from tests.unit.conftest import make_input_element_mock
-from tests.unit.helpers import make_organization, make_step, make_task
-
-_NOW = datetime.now(UTC)
-_ORG = make_organization(_NOW)
-_TASK = make_task(_NOW, _ORG, navigation_payload={}, navigation_goal="Fill the postal code field")
-_STEP = make_step(_NOW, _TASK, step_id="stp-masked-readback", status=StepStatus.created, order=0, output=None)
-
-
-def test_mask_evidence_from_pattern() -> None:
- assert _has_input_mask_evidence(pattern=r"\d{5}(-\d{4})?", placeholder=None) is True
-
-
-def test_mask_evidence_from_placeholder_slots() -> None:
- assert _has_input_mask_evidence(pattern=None, placeholder="_____") is True
- assert _has_input_mask_evidence(pattern=None, placeholder="___-__-____") is True
-
-
-def test_no_mask_evidence_for_plain_field() -> None:
- assert _has_input_mask_evidence(pattern=None, placeholder="Enter ZIP") is False
- assert _has_input_mask_evidence(pattern=None, placeholder=None) is False
-
-
-def test_incomplete_when_commit_is_empty() -> None:
- assert _committed_value_is_masked_incomplete("12345", "") is True
- assert _committed_value_is_masked_incomplete("12345", None) is True
-
-
-def test_incomplete_when_only_placeholders_remain() -> None:
- assert _committed_value_is_masked_incomplete("12345", "_____") is True
- assert _committed_value_is_masked_incomplete("12345", "123__") is True
-
-
-def test_complete_when_value_committed() -> None:
- assert _committed_value_is_masked_incomplete("12345", "12345") is False
- # A ZIP+4 field that reformats the committed value is complete, not incomplete.
- assert _committed_value_is_masked_incomplete("12345", "12345-6789") is False
-
-
-def test_placeholder_char_inside_complete_value_is_not_incomplete() -> None:
- # An underscore that is part of a fully-typed value must not read as an unfilled mask slot.
- assert _committed_value_is_masked_incomplete("foo_bar", "foo_bar") is False
-
-
-def test_nothing_expected_is_never_incomplete() -> None:
- assert _committed_value_is_masked_incomplete("", "") is False
- assert _committed_value_is_masked_incomplete("---", "_____") is False
-
-
-def _mock_element(input_values: list[str | None]) -> MagicMock:
- locator = MagicMock()
- locator.input_value = AsyncMock(side_effect=input_values)
- element = MagicMock()
- element.get_locator = MagicMock(return_value=locator)
- return element
-
-
-def _mock_frame() -> MagicMock:
- frame = MagicMock()
- frame.safe_wait_for_animation_end = AsyncMock()
- return frame
-
-
-async def test_masked_readback_raises_on_empty_commit() -> None:
- element = _mock_element(["", ""])
- with pytest.raises(MaskedInputReadbackMismatch) as exc:
- await _verify_masked_input_after_fill(
- skyvern_element=element,
- skyvern_frame=_mock_frame(),
- tag_name="input",
- expected_value="12345",
- known_masked=True,
- )
- assert exc.value.expected_char_count == 5
- assert exc.value.committed_char_count == 0
-
-
-async def test_masked_readback_accepts_committed_value() -> None:
- element = _mock_element(["12345"])
- await _verify_masked_input_after_fill(
- skyvern_element=element,
- skyvern_frame=_mock_frame(),
- tag_name="input",
- expected_value="12345",
- known_masked=True,
- )
-
-
-async def test_masked_readback_settles_before_declaring_failure() -> None:
- # First read is still empty, the value lands after the settle wait — no failure.
- element = _mock_element(["", "12345"])
- await _verify_masked_input_after_fill(
- skyvern_element=element,
- skyvern_frame=_mock_frame(),
- tag_name="input",
- expected_value="12345",
- known_masked=True,
- )
-
-
-async def test_masked_readback_skips_field_without_mask_signal() -> None:
- # No attribute evidence and no placeholder chars in the commit: leave the field alone even if empty.
- element = _mock_element([""])
- await _verify_masked_input_after_fill(
- skyvern_element=element,
- skyvern_frame=_mock_frame(),
- tag_name="input",
- expected_value="12345",
- known_masked=False,
- )
-
-
-async def test_masked_readback_triggers_on_placeholder_commit_without_attr_evidence() -> None:
- element = _mock_element(["_____", "_____"])
- with pytest.raises(MaskedInputReadbackMismatch):
- await _verify_masked_input_after_fill(
- skyvern_element=element,
- skyvern_frame=_mock_frame(),
- tag_name="input",
- expected_value="12345",
- known_masked=False,
- )
-
-
-async def _run_masked_input_text(text: str, read_backs: list[str | None]) -> list:
- element = make_input_element_mock()
- dom_instance = MagicMock()
- dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=element)
-
- inc = MagicMock()
- inc.start_listen_dom_increment = AsyncMock()
- inc.stop_listen_dom_increment = AsyncMock()
- inc.get_incremental_element_tree = AsyncMock(return_value=[])
-
- skyvern_frame = MagicMock()
- skyvern_frame.safe_wait_for_animation_end = AsyncMock()
-
- scraped_page = MagicMock()
- scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}
-
- with (
- patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),
- patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),
- patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(side_effect=read_backs)),
- patch("skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task", return_value=text),
- patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=None)),
- patch("skyvern.webeye.actions.handler._is_masked_input_readback_fix_enabled", new=AsyncMock(return_value=True)),
- ):
- return await handle_input_text_action(
- action=InputTextAction(element_id="AADC", text=text, reasoning="fill postal code"),
- page=MagicMock(),
- scraped_page=scraped_page,
- task=_TASK,
- step=_STEP,
- )
-
-
-async def test_masked_retype_swallowed_by_mask_fails_loudly() -> None:
- # The field advertises no mask attrs, so the only mask evidence is the placeholder chars seen on the
- # first read-back. The retype is then swallowed whole and reads back empty: without carrying that
- # evidence into the retry, the empty commit looks like an unmasked field and passes silently.
- results = await _run_masked_input_text("12345", ["", "12_45", "12_45", "", ""])
-
- assert len(results) == 1
- assert isinstance(results[0], ActionFailure)
- assert results[0].exception_type == MaskedInputReadbackMismatch.__name__
diff --git a/tests/unit/test_input_text_phone_readback.py b/tests/unit/test_input_text_phone_readback.py
index 1e88a6b72..9ec6afc57 100644
--- a/tests/unit/test_input_text_phone_readback.py
+++ b/tests/unit/test_input_text_phone_readback.py
@@ -3,17 +3,13 @@ from __future__ import annotations
from unittest.mock import AsyncMock, MagicMock
import pytest
-from playwright._impl._errors import Error as PlaywrightError
-from skyvern.exceptions import InputTextCommitMismatch, PhoneNumberInputMismatch
+from skyvern.exceptions import PhoneNumberInputMismatch
from skyvern.webeye.actions.handler import (
_fill_nanp_tel_with_readback,
_log_tel_fallback_fill_digit_counts,
- _verify_generic_input_commit,
- input_text_values_match,
verify_phone_input_digits,
)
-from skyvern.webeye.actions.responses import ActionFailure
NANP_DIGITS = "2245550199"
NANP_E164 = f"+1{NANP_DIGITS}"
@@ -214,328 +210,3 @@ async def test_tel_fallback_digit_count_log_swallows_read_errors() -> None:
task_id="tsk_1",
step_id="stp_1",
)
-
-
- ("expected", "actual", "input_type"),
- [
- ("5551234567", "(555) 123-4567", "text"),
- ("12345", "12345-____", "text"),
- ("ab12 cd", "AB12-CD", "search"),
- ("postal code", "POSTAL_CODE", "textarea"),
- ],
-)
-def test_input_text_values_match_ignores_plain_text_mask_formatting(
- expected: str, actual: str, input_type: str
-) -> None:
- assert input_text_values_match(expected, actual, input_type=input_type)
-
-
- ("expected", "actual", "input_type"),
- [
- ("[email protected]", "user@exampletest", "email"),
- ("https://example.test/path", "https://exampletest/path", "url"),
- ("-12.5", "125", "number"),
- ],
-)
-def test_input_text_values_match_preserves_semantic_punctuation(expected: str, actual: str, input_type: str) -> None:
- assert not input_text_values_match(expected, actual, input_type=input_type)
-
-
-def test_input_text_values_match_trims_semantic_input_whitespace() -> None:
- assert input_text_values_match(" [email protected] ", "[email protected]", input_type="email")
-
-
- ("expected", "actual", "committed", "why"),
- [
- # A field has committed when its value derives from what was typed.
- ("12345", "12345", True, "exact"),
- ("1234567890", "(123) 456-7890", True, "reformat only rearranges separators"),
- ("12345", "12345-6789", True, "widget enriches the typed value"),
- ("foo_bar", "foo_bar", True, "placeholder char inside a fully typed value"),
- # Only these are failures.
- ("12345", "", False, "empty"),
- ("12345", "_____", False, "mask kept none of the keystrokes"),
- ("12345", "123__", False, "mask kept only some of them"),
- ("12345", "99999", False, "genuinely different value"),
- ("12345", "999123459", False, "contains the typed value but does not derive from it"),
- ("---", "___", True, "separators typed, separators back: a reformat of nothing"),
- ("---", "xyz", False, "separators typed, a real value back: not a reformat of it"),
- ],
-)
-def test_input_text_values_match_treats_derived_values_as_committed(
- expected: str, actual: str, committed: bool, why: str
-) -> None:
- assert input_text_values_match(expected, actual, input_type="text") is committed, why
-
-
-def test_input_text_values_match_rejects_empty_commit() -> None:
- assert not input_text_values_match("-", "", input_type="text")
-
-
-def _make_commit_element(readbacks: list[str]) -> MagicMock:
- locator = MagicMock()
- locator.input_value = AsyncMock(side_effect=readbacks)
- locator.evaluate = AsyncMock()
- element = _make_element(locator)
- element.input_clear = AsyncMock()
- element.input_sequentially = AsyncMock()
- element.input_fill = AsyncMock()
- return element
-
-
-def _make_commit_frame() -> MagicMock:
- frame = MagicMock()
- frame.safe_wait_for_animation_end = AsyncMock()
- return frame
-
-
-async def test_generic_commit_verification_accepts_initial_readback() -> None:
- element = _make_commit_element(["committed"])
- dom = MagicMock()
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=_make_commit_frame(),
- dom=dom,
- tag_name="input",
- input_type="text",
- expected_value="committed",
- )
-
- assert result is None
- element.input_clear.assert_not_awaited()
- element.input_fill.assert_not_awaited()
- element.get_locator().evaluate.assert_not_awaited()
-
-
-async def test_generic_commit_verification_settles_before_destroying_a_slow_value() -> None:
- # A masked or debounced field can land its value a beat after the keystrokes settle. Escalating on
- # the first read-back would clear and retype a field that was about to commit correctly.
- element = _make_commit_element(["", "committed"])
- frame = _make_commit_frame()
- dom = MagicMock()
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=frame,
- dom=dom,
- tag_name="input",
- input_type="text",
- expected_value="committed",
- )
-
- assert result is None
- frame.safe_wait_for_animation_end.assert_awaited_once()
- element.input_clear.assert_not_awaited()
- element.input_sequentially.assert_not_awaited()
-
-
-async def test_generic_commit_verification_escalates_to_fill() -> None:
- element = _make_commit_element(["", "", "", "committed"])
- dom = MagicMock()
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=_make_commit_frame(),
- dom=dom,
- tag_name="input",
- input_type="text",
- expected_value="committed",
- )
-
- assert result is None
- element.input_clear.assert_awaited_once()
- element.input_sequentially.assert_awaited_once_with(text="committed")
- element.input_fill.assert_awaited_once_with(text="committed")
- element.get_locator().evaluate.assert_not_awaited()
-
-
-async def test_generic_commit_verification_escalates_to_native_setter() -> None:
- element = _make_commit_element(["", "", "", "", "committed"])
- dom = MagicMock()
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=_make_commit_frame(),
- dom=dom,
- tag_name="textarea",
- input_type="textarea",
- expected_value="committed",
- )
-
- assert result is None
- element.input_fill.assert_awaited_once_with(text="committed")
- element.get_locator().evaluate.assert_awaited_once()
- assert element.get_locator().evaluate.await_args.args[1] == "committed"
-
-
-async def test_generic_commit_verification_returns_typed_failure_without_value() -> None:
- element = _make_commit_element(["", "", "", "", ""])
- dom = MagicMock()
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=_make_commit_frame(),
- dom=dom,
- tag_name="input",
- input_type="text",
- expected_value="sensitive-value",
- )
-
- assert isinstance(result, ActionFailure)
- assert result.exception_type == InputTextCommitMismatch.__name__
- assert "expected length 14" in (result.exception_message or "")
- assert "actual length 0" in (result.exception_message or "")
- assert "sensitive-value" not in (result.exception_message or "")
-
-
-async def test_generic_commit_verification_skips_navigation_during_escalation() -> None:
- element = _make_commit_element(["", ""])
- dom = MagicMock()
- element.input_clear.side_effect = PlaywrightError("Execution context was destroyed during navigation")
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=_make_commit_frame(),
- dom=dom,
- tag_name="input",
- input_type="text",
- expected_value="committed",
- )
-
- assert result is None
-
-
-async def test_generic_commit_verification_reresolves_detached_element() -> None:
- detached_element = _make_commit_element([])
- detached_element.get_locator().input_value.side_effect = PlaywrightError("Element is not attached to the DOM")
- replacement_element = _make_commit_element(["committed"])
- dom = MagicMock()
- dom.get_skyvern_element_by_id = AsyncMock(return_value=replacement_element)
-
- result = await _verify_generic_input_commit(
- skyvern_element=detached_element,
- skyvern_frame=_make_commit_frame(),
- dom=dom,
- tag_name="input",
- input_type="text",
- expected_value="committed",
- )
-
- assert result is None
- dom.get_skyvern_element_by_id.assert_awaited_once_with(detached_element.get_id())
-
-
-async def test_generic_commit_verification_accepts_plausible_autocomplete_rewrite() -> None:
- element = _make_commit_element(["committed option"])
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=_make_commit_frame(),
- dom=MagicMock(),
- tag_name="input",
- input_type="text",
- expected_value="committed",
- allow_autocomplete_rewrite=True,
- )
-
- assert result is None
- element.input_clear.assert_not_awaited()
-
-
-async def test_generic_commit_verification_rejects_truncated_value_as_autocomplete_rewrite() -> None:
- element = _make_commit_element(["123", "123", "123", "123", "123"])
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=_make_commit_frame(),
- dom=MagicMock(),
- tag_name="input",
- input_type="text",
- expected_value="12345",
- allow_autocomplete_rewrite=True,
- )
-
- assert isinstance(result, ActionFailure)
- assert result.exception_type == InputTextCommitMismatch.__name__
- element.input_clear.assert_awaited_once()
- element.input_fill.assert_awaited_once_with(text="12345")
-
-
-async def test_generic_commit_verification_escalates_garbled_readback_without_rewrite_allowance() -> None:
- element = _make_commit_element(["123", "123", "12345"])
-
- result = await _verify_generic_input_commit(
- skyvern_element=element,
- skyvern_frame=_make_commit_frame(),
- dom=MagicMock(),
- tag_name="input",
- input_type="text",
- expected_value="12345",
- )
-
- assert result is None
- element.input_clear.assert_awaited_once()
- element.input_sequentially.assert_awaited_once_with(text="12345")
-
-
-async def test_generic_commit_verification_reresolves_once_per_escalation_stage() -> None:
- first_element = _make_commit_element([])
- first_element.get_locator().input_value.side_effect = PlaywrightError("Element is not attached to the DOM")
- second_element = _make_commit_element([])
- second_element.get_locator().input_value.side_effect = [
- "",
- "",
- PlaywrightError("Element is not attached to the DOM"),
- ]
- third_element = _make_commit_element(["committed"])
- dom = MagicMock()
- dom.get_skyvern_element_by_id = AsyncMock(side_effect=[second_element, third_element])
-
- result = await _verify_generic_input_commit(
- skyvern_element=first_element,
- skyvern_frame=_make_commit_frame(),
- dom=dom,
- tag_name="input",
- input_type="text",
- expected_value="committed",
- )
-
- assert result is None
- assert dom.get_skyvern_element_by_id.await_count == 2
- second_element.input_sequentially.assert_awaited_once_with(text="committed")
-
-
-async def test_generic_commit_verification_soft_skips_detach_during_fill_stage() -> None:
- detached_element = _make_commit_element([])
- detached_element.get_locator().input_value.side_effect = PlaywrightError("Element is not attached to the DOM")
- replacement_element = _make_commit_element(["", ""])
- replacement_element.input_clear.side_effect = PlaywrightError("Element is not attached to the DOM")
- dom = MagicMock()
- dom.get_skyvern_element_by_id = AsyncMock(return_value=replacement_element)
-
- result = await _verify_generic_input_commit(
- skyvern_element=detached_element,
- skyvern_frame=_make_commit_frame(),
- dom=dom,
- tag_name="input",
- input_type="text",
- expected_value="committed",
- )
-
- assert result is None
diff --git a/tests/unit/test_input_text_search_bar_skip.py b/tests/unit/test_input_text_search_bar_skip.py
index edb9edb90..c666663d9 100644
--- a/tests/unit/test_input_text_search_bar_skip.py
+++ b/tests/unit/test_input_text_search_bar_skip.py
@@ -9,19 +9,16 @@ and non-clobbering follow-ups (Tab/Escape/Arrow, different-element actions) are
from __future__ import annotations
-import copy
from datetime import UTC, datetime
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from playwright.async_api import Error as PlaywrightError
-from skyvern.exceptions import AutoCompletionCommitFailure
from skyvern.forge.sdk.models import StepStatus
from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextAction, KeypressAction
-from skyvern.webeye.actions.handler import AUTOCOMPLETE_COMMIT_REQUIRED_FLAG, handle_input_text_action
+from skyvern.webeye.actions.handler import handle_input_text_action
from skyvern.webeye.actions.handler_utils import keys_include_enter, should_stop_batch_after_dropdown_select
-from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
+from skyvern.webeye.actions.responses import ActionSuccess
from tests.unit.conftest import make_input_element_mock
from tests.unit.helpers import make_organization, make_step, make_task
@@ -122,94 +119,12 @@ async def _run_search_bar_input(stop_flag: bool, incremental: list[dict]) -> lis
"skyvern.webeye.actions.handler.sequentially_select_from_dropdown",
new=AsyncMock(return_value=select_result),
),
- patch(
- "skyvern.webeye.actions.handler._is_input_text_commit_verification_enabled",
- new=AsyncMock(return_value=False),
- ),
):
return await handle_input_text_action(
action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP
)
-def _fake_experimentation_provider(enabled_flags: set[str]) -> MagicMock:
- provider = MagicMock()
-
- async def _is_enabled(flag: str, distinct_id: str, properties: dict | None = None) -> bool:
- return flag in enabled_flags
-
- provider.is_feature_enabled_cached = AsyncMock(side_effect=_is_enabled)
- provider.resolve_feature_enabled_unrecorded = AsyncMock(return_value=False)
- return provider
-
-
-async def _run_autocomplete_input_without_suggestions(
- *,
- is_search_bar: bool,
- is_location_input: bool,
- commit_flag_enabled: bool,
-) -> tuple[list, MagicMock]:
- """Drive handle_input_text_action through the REAL autocomplete chain with a widget
- that never renders suggestions."""
- skyvern_el = make_input_element_mock(element_id="AADC")
- skyvern_el.is_auto_completion_input = AsyncMock(return_value=True)
- skyvern_el.press_fill = AsyncMock()
- skyvern_el.has_attr = AsyncMock(return_value=False)
- skyvern_el.get_locator.return_value.click = AsyncMock()
-
- dom_instance = MagicMock()
- dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)
-
- inc = MagicMock()
- inc.start_listen_dom_increment = AsyncMock()
- inc.stop_listen_dom_increment = AsyncMock()
- inc.get_incremental_elements_num = AsyncMock(return_value=0)
- inc.get_incremental_element_tree = AsyncMock(return_value=[])
- inc.build_html_tree.return_value = "<div></div>"
- inc.id_to_element_dict = {}
-
- skyvern_frame = MagicMock()
- skyvern_frame.safe_wait_for_animation_end = AsyncMock()
-
- scraped_page = MagicMock()
- scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}
- scraped_page.id_to_css_dict = {"AADC": "#aadc"}
- scraped_after = MagicMock()
- scraped_after.id_to_css_dict = {"AADC": "#aadc"}
- scraped_page.generate_scraped_page_without_screenshots = AsyncMock(return_value=scraped_after)
-
- context = InputOrSelectContext(field="Account", is_search_bar=is_search_bar, is_location_input=is_location_input)
- enabled_flags = {AUTOCOMPLETE_COMMIT_REQUIRED_FLAG} if commit_flag_enabled else set()
-
- action = _input()
-
- with (
- patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),
- patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),
- patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(return_value="")),
- patch(
- "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",
- return_value="123456",
- ),
- patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),
- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- ):
- mock_app.EXPERIMENTATION_PROVIDER = _fake_experimentation_provider(enabled_flags)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value={})
- mock_app.AGENT_FUNCTION = MagicMock()
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
- results = await handle_input_text_action(
- action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP
- )
- return results, skyvern_el
-
-
@pytest.mark.asyncio
async def test_selection_with_clobbering_next_stops_batch() -> None:
"""Selection committed + a clobbering next action (flag set) -> skip_remaining_actions=True."""
@@ -232,128 +147,3 @@ async def test_plain_search_input_does_not_stop_batch() -> None:
results = await _run_search_bar_input(stop_flag=True, incremental=[])
assert len(results) == 1 and isinstance(results[0], ActionSuccess)
assert not results[0].skip_remaining_actions
-
-
-async def test_search_bar_without_suggestions_falls_through_to_plain_typing() -> None:
- """A search bar whose autocomplete never renders suggestions must still be typed into (flag off)."""
- results, skyvern_el = await _run_autocomplete_input_without_suggestions(
- is_search_bar=True,
- is_location_input=False,
- commit_flag_enabled=False,
- )
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- skyvern_el.input_sequentially.assert_awaited_once()
-
-
-async def test_location_field_without_suggestions_falls_through_when_flag_off() -> None:
- results, skyvern_el = await _run_autocomplete_input_without_suggestions(
- is_search_bar=False,
- is_location_input=True,
- commit_flag_enabled=False,
- )
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- skyvern_el.input_sequentially.assert_awaited_once()
-
-
-async def test_search_bar_direct_search_enter_navigation_is_implicit_commit() -> None:
- """direct_searching presses Enter, the page navigates, and every post-Enter read-back hits a stale
- element; the action must succeed as an implicit commit without falling through to plain typing."""
- skyvern_el = make_input_element_mock(element_id="AADC")
- skyvern_el.is_auto_completion_input = AsyncMock(return_value=True)
- skyvern_el.press_fill = AsyncMock()
- skyvern_el.has_attr = AsyncMock(return_value=False)
-
- enter_pressed = False
-
- async def _press_key(key: str, **_: object) -> None:
- nonlocal enter_pressed
- if key == "Enter":
- enter_pressed = True
-
- skyvern_el.press_key = AsyncMock(side_effect=_press_key)
-
- async def _read_input_value(tag_name: str, locator: object) -> str:
- if enter_pressed:
- raise PlaywrightError("Element is not attached to the DOM")
- return ""
-
- suggestions = [
- {"id": "OPT1", "tagName": "li", "attributes": {"role": "option"}, "text": "Target 123456"},
- {"id": "OPT2", "tagName": "li", "attributes": {"role": "option"}, "text": "Target 654321"},
- ]
-
- dom_instance = MagicMock()
- dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)
-
- inc = MagicMock()
- inc.start_listen_dom_increment = AsyncMock()
- inc.stop_listen_dom_increment = AsyncMock()
- inc.get_incremental_elements_num = AsyncMock(return_value=len(suggestions))
- inc.get_incremental_element_tree = AsyncMock(return_value=copy.deepcopy(suggestions))
- inc.build_html_tree.return_value = "<div>options</div>"
- inc.id_to_element_dict = {}
-
- scraped_page = MagicMock()
- scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}
- scraped_page.id_to_css_dict = {"AADC": "#aadc"}
-
- context = InputOrSelectContext(field="Search", is_search_bar=True, is_location_input=False)
- llm_response = {
- "auto_completion_attempt": False,
- "relevance_float": 0.0,
- "id": "",
- "direct_searching": True,
- "value": "123456",
- "reasoning": "Search directly",
- }
-
- action = _input()
-
- with (
- patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),
- patch(
- "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
- new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
- ),
- patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),
- patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(side_effect=_read_input_value)),
- patch(
- "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",
- return_value="123456",
- ),
- patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),
- patch("skyvern.webeye.actions.handler.asyncio.sleep", new=AsyncMock()),
- patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
- patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
- patch("skyvern.webeye.actions.handler.app") as mock_app,
- ):
- mock_app.EXPERIMENTATION_PROVIDER = _fake_experimentation_provider(set())
- mock_app.AUTO_COMPLETION_LLM_API_HANDLER = AsyncMock(return_value=llm_response)
- mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": []})
- mock_app.AGENT_FUNCTION = MagicMock()
- mock_prompt.load_prompt.return_value = "mocked prompt"
- mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
- results = await handle_input_text_action(
- action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP
- )
-
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- skyvern_el.press_key.assert_awaited_once_with("Enter")
- skyvern_el.input_sequentially.assert_not_awaited()
-
-
-async def test_location_field_without_commit_fails_when_flag_on() -> None:
- """With the commit-required flag on, a location field never falls through to plain typing."""
- results, skyvern_el = await _run_autocomplete_input_without_suggestions(
- is_search_bar=False,
- is_location_input=True,
- commit_flag_enabled=True,
- )
- assert len(results) == 1 and isinstance(results[0], ActionFailure)
- assert results[0].exception_type == AutoCompletionCommitFailure.__name__
- assert "suggestions_never_rendered" in (results[0].exception_message or "")
- skyvern_el.input_sequentially.assert_not_awaited()
diff --git a/tests/unit/test_input_text_tel_card_routing.py b/tests/unit/test_input_text_tel_card_routing.py
index c431a0552..085e333af 100644
--- a/tests/unit/test_input_text_tel_card_routing.py
+++ b/tests/unit/test_input_text_tel_card_routing.py
@@ -1,7 +1,7 @@
from __future__ import annotations
from datetime import UTC, datetime
-from unittest.mock import ANY, AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -55,11 +55,9 @@ async def _run_input_text(
resolved: str | None = None,
tel_fix_enabled: bool = True,
tel_verify_side_effect: list[Exception | None] | None = None,
- generic_flag: bool = True,
- incremental_elements: list[dict] | None = None,
tag_name: str = "input",
blocker: MagicMock | None = None,
-) -> tuple[list, AsyncMock, AsyncMock, AsyncMock, MagicMock, AsyncMock, AsyncMock]:
+) -> tuple[list, AsyncMock, AsyncMock, AsyncMock, MagicMock, AsyncMock]:
dom_instance = MagicMock()
dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=el)
if blocker is not None:
@@ -69,7 +67,7 @@ async def _run_input_text(
inc = MagicMock()
inc.start_listen_dom_increment = AsyncMock()
inc.stop_listen_dom_increment = AsyncMock()
- inc.get_incremental_element_tree = AsyncMock(return_value=incremental_elements or [])
+ inc.get_incremental_element_tree = AsyncMock(return_value=[])
skyvern_frame = MagicMock()
skyvern_frame.safe_wait_for_animation_end = AsyncMock()
@@ -79,7 +77,6 @@ async def _run_input_text(
card_readback = AsyncMock(return_value=None)
tel_verify = AsyncMock(side_effect=tel_verify_side_effect)
- generic_verify = AsyncMock(return_value=None)
phone_format = AsyncMock(return_value=text)
warning_log = MagicMock()
secret_readback = AsyncMock(return_value=None)
@@ -97,16 +94,10 @@ async def _run_input_text(
),
patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=None)),
patch("skyvern.webeye.actions.handler._is_tel_digit_fix_enabled", new=AsyncMock(return_value=tel_fix_enabled)),
- patch(
- "skyvern.webeye.actions.handler._is_input_text_commit_verification_enabled",
- new=AsyncMock(return_value=generic_flag),
- ),
patch("skyvern.webeye.actions.handler.check_phone_number_format", new=phone_format),
patch("skyvern.webeye.actions.handler._fill_card_number_with_readback", new=card_readback),
patch("skyvern.webeye.actions.handler._fill_secret_with_readback", new=secret_readback),
patch("skyvern.webeye.actions.handler._verify_tel_input_after_fill", new=tel_verify),
- patch("skyvern.webeye.actions.handler._verify_generic_input_commit", new=generic_verify),
- patch("skyvern.webeye.actions.handler.check_date_format", new=AsyncMock(return_value=text)),
patch("skyvern.webeye.actions.handler.LOG.warning", new=warning_log),
):
results = await handle_input_text_action(
@@ -117,7 +108,7 @@ async def _run_input_text(
step=_STEP,
)
- return results, card_readback, tel_verify, phone_format, warning_log, generic_verify, secret_readback
+ return results, card_readback, tel_verify, phone_format, warning_log, secret_readback
@pytest.mark.asyncio
@@ -131,9 +122,7 @@ async def _run_input_text(
async def test_tel_card_number_field_uses_card_readback_not_phone_format(attrs: dict[str, str | None]) -> None:
el = _mock_input(attrs)
- results, card_readback, tel_verify, phone_format, _, generic_verify, secret_readback = await _run_input_text(
- el, VISA_16
- )
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, VISA_16)
assert len(results) == 1 and isinstance(results[0], ActionSuccess)
card_readback.assert_awaited_once_with(
@@ -144,7 +133,6 @@ async def test_tel_card_number_field_uses_card_readback_not_phone_format(attrs:
)
phone_format.assert_not_awaited()
tel_verify.assert_not_awaited()
- generic_verify.assert_not_awaited()
secret_readback.assert_not_awaited()
el.input_sequentially.assert_not_awaited()
@@ -153,9 +141,7 @@ async def test_tel_card_number_field_uses_card_readback_not_phone_format(attrs:
async def test_ten_digit_tel_phone_uses_tel_readback_not_card_readback() -> None:
el = _mock_input({"type": "tel", "autocomplete": None, "name": "phone"})
- results, card_readback, tel_verify, phone_format, _, generic_verify, secret_readback = await _run_input_text(
- el, "224-555-0199"
- )
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(el, "224-555-0199")
assert len(results) == 1 and isinstance(results[0], ActionSuccess)
el.input_sequentially.assert_awaited_once_with(text="2245550199")
@@ -169,7 +155,6 @@ async def test_ten_digit_tel_phone_uses_tel_readback_not_card_readback() -> None
)
card_readback.assert_not_awaited()
phone_format.assert_not_awaited()
- generic_verify.assert_not_awaited()
secret_readback.assert_not_awaited()
@@ -177,7 +162,7 @@ async def test_ten_digit_tel_phone_uses_tel_readback_not_card_readback() -> None
async def test_tel_flag_off_preserves_legacy_format_and_sequential_fill() -> None:
el = _mock_input({"type": "tel", "autocomplete": None, "name": "phone"})
- results, _, tel_verify, phone_format, _, _, _ = await _run_input_text(
+ results, _, tel_verify, phone_format, _, _ = await _run_input_text(
el,
"224-555-0199",
tel_fix_enabled=False,
@@ -199,7 +184,7 @@ async def test_separator_only_tel_never_forces_nanp_country_code() -> None:
PhoneNumberInputMismatch(expected_digit_count=10, actual_digit_count=12),
]
- results, _, tel_verify, _, warning_log, _, _ = await _run_input_text(
+ results, _, tel_verify, _, warning_log, _ = await _run_input_text(
el,
"224-555-0199",
tel_verify_side_effect=mismatches,
@@ -227,7 +212,7 @@ async def test_explicit_nanp_tel_keeps_constraint_safe_e164_fallback() -> None:
None,
]
- results, _, tel_verify, _, _, _, _ = await _run_input_text(
+ results, _, tel_verify, _, _, _ = await _run_input_text(
el,
"+1 (224) 555-0199",
tel_verify_side_effect=mismatches_then_success,
@@ -243,43 +228,6 @@ async def test_explicit_nanp_tel_keeps_constraint_safe_e164_fallback() -> None:
]
-async def test_generic_text_input_runs_commit_verification() -> None:
- el = _mock_input({"type": "text", "autocomplete": None, "name": "postal_code"})
-
- results, card_readback, tel_verify, phone_format, _, generic_verify, _ = await _run_input_text(el, "12345")
-
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- el.input_sequentially.assert_awaited_once_with(text="12345")
- generic_verify.assert_awaited_once_with(
- skyvern_element=el,
- skyvern_frame=ANY,
- dom=ANY,
- tag_name="input",
- input_type="text",
- expected_value="12345",
- allow_autocomplete_rewrite=False,
- )
- card_readback.assert_not_awaited()
- tel_verify.assert_not_awaited()
- phone_format.assert_not_awaited()
-
-
-async def test_blocking_date_input_refreshes_type_before_routing() -> None:
- original = _mock_input({"type": "text", "autocomplete": None, "name": "start_date"})
- blocking = _mock_input({"type": "date", "autocomplete": None, "name": "start_date"})
- blocking.get_id.return_value = "BLOCKING"
- original.find_blocking_element = AsyncMock(return_value=(blocking, True))
-
- results, _, _, _, _, generic_verify, _ = await _run_input_text(original, "2026-07-17")
-
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- blocking.input_fill.assert_awaited_once_with(text="2026-07-17")
- blocking.input_sequentially.assert_not_awaited()
- generic_verify.assert_not_awaited()
-
-
@pytest.mark.asyncio
async def test_blocking_tel_input_rechecks_constraints_before_readback() -> None:
original = _mock_input({"type": "tel", "autocomplete": None, "name": "phone"})
@@ -288,7 +236,7 @@ async def test_blocking_tel_input_rechecks_constraints_before_readback() -> None
)
blocking.get_id.return_value = "BLOCKING"
- results, _, tel_verify, _, _, _, _ = await _run_input_text(original, "+1 (224) 555-0199", blocker=blocking)
+ results, _, tel_verify, _, _, _ = await _run_input_text(original, "+1 (224) 555-0199", blocker=blocking)
assert len(results) == 1 and isinstance(results[0], ActionSuccess)
blocking.input_sequentially.assert_awaited_once_with(text="2245550199")
@@ -303,67 +251,6 @@ async def test_blocking_tel_input_rechecks_constraints_before_readback() -> None
)
-async def test_generic_text_input_skips_commit_verification_when_flag_disabled() -> None:
- el = _mock_input({"type": "text", "autocomplete": None, "name": "postal_code"})
-
- results, _, _, _, _, generic_verify, _ = await _run_input_text(el, "12345", generic_flag=False)
-
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- el.input_sequentially.assert_awaited_once_with(text="12345")
- generic_verify.assert_not_awaited()
- # The credential-scope classification and the baseline live_input_type each read the live type,
- # independent of the commit-verification flag; a disabled flag still queries type exactly twice.
- type_queries = [call for call in el.get_attr.await_args_list if call.args == ("type",)]
- assert len(type_queries) == 2
-
-
-async def test_generic_text_input_verifies_after_unrelated_dom_mutation() -> None:
- el = _mock_input({"type": "text", "autocomplete": None, "name": "postal_code"})
-
- results, _, _, _, _, generic_verify, _ = await _run_input_text(
- el,
- "12345",
- incremental_elements=[{"tagName": "div", "attributes": {"class": "field-error"}, "children": []}],
- )
-
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- generic_verify.assert_awaited_once()
- assert generic_verify.await_args.kwargs["allow_autocomplete_rewrite"] is False
-
-
-async def test_generic_text_input_allows_autocomplete_rewrite_for_option_mutation() -> None:
- el = _mock_input({"type": "text", "autocomplete": None, "name": "postal_code"})
-
- results, _, _, _, _, generic_verify, _ = await _run_input_text(
- el,
- "12345",
- incremental_elements=[
- {
- "tagName": "ul",
- "attributes": {"role": "listbox"},
- "children": [{"tagName": "li", "attributes": {"role": "option"}, "children": []}],
- }
- ],
- )
-
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- generic_verify.assert_awaited_once()
- assert generic_verify.await_args.kwargs["allow_autocomplete_rewrite"] is True
-
-
-async def test_password_input_skips_commit_verification() -> None:
- el = _mock_input({"type": "password", "autocomplete": None, "name": "credential"})
-
- results, _, _, _, _, generic_verify, _ = await _run_input_text(el, "private payload")
-
- assert len(results) == 1 and isinstance(results[0], ActionSuccess)
- generic_verify.assert_not_awaited()
-
-
@pytest.mark.asyncio
async def test_secret_tel_value_uses_tel_verifier_not_secret_readback() -> None:
# A resolved secret that is a NANP phone number must keep the digit-normalized tel verification
@@ -371,7 +258,7 @@ async def test_secret_tel_value_uses_tel_verifier_not_secret_readback() -> None:
# the bare digits against the formatted value and fail a correct fill.
el = _mock_input({"type": "tel", "autocomplete": None, "name": "phone"})
- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "{{ phone }}", resolved="224-555-0199"
)
@@ -395,7 +282,7 @@ async def test_single_character_secret_skips_readback() -> None:
# (e.g. a multi-field TOTP digit routed into a masked box: is_secret_value True, is_totp_value False).
el = _mock_input({"type": "password", "autocomplete": None, "name": "otp-digit"})
- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "{{ digit }}", resolved="5"
)
@@ -413,7 +300,7 @@ async def test_secret_in_non_input_element_skips_readback() -> None:
el = _mock_input({"type": None, "autocomplete": None, "name": "note"})
el.get_tag_name.return_value = "div"
- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "{{ sec }}", resolved="mysecretvalue", tag_name="div"
)
@@ -429,7 +316,7 @@ async def test_secret_in_exact_value_input_uses_readback(input_type: str | None)
# its .value exactly, so the credential read-back verifier runs and is told the live type.
el = _mock_input({"type": input_type, "autocomplete": None, "name": "credential"})
- results, card_readback, tel_verify, phone_format, _, generic_verify, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "{{ sec }}", resolved="mysecretvalue", tag_name="input"
)
@@ -442,8 +329,6 @@ async def test_secret_in_exact_value_input_uses_readback(input_type: str | None)
maxlength=None,
)
el.input_sequentially.assert_not_awaited()
- # A credential takes only the exact secret read-back; the generic commit verifier must not also run.
- generic_verify.assert_not_awaited()
@pytest.mark.asyncio
@@ -454,7 +339,7 @@ async def test_secret_in_non_exact_value_input_skips_readback(input_type: str) -
# path earlier and never reaches this gate.)
el = _mock_input({"type": input_type, "autocomplete": None, "name": "field"})
- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "{{ sec }}", resolved="mysecretvalue", tag_name="input"
)
@@ -471,7 +356,7 @@ async def test_secret_readback_skips_when_retargeted_to_out_of_scope_blocker() -
el = _mock_input({"type": "text", "autocomplete": None, "name": "credential"})
blocker = _mock_input({"type": "number", "autocomplete": None, "name": "overlay"})
- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "{{ sec }}", resolved="mysecretvalue", tag_name="input", blocker=blocker
)
@@ -488,7 +373,7 @@ async def test_secret_readback_runs_on_retargeted_element_type() -> None:
el = _mock_input({"type": "text", "autocomplete": None, "name": "overlay"})
blocker = _mock_input({"type": "password", "autocomplete": None, "name": "password"})
- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "{{ sec }}", resolved="mysecretvalue", tag_name="input", blocker=blocker
)
@@ -508,7 +393,7 @@ async def test_non_secret_exact_value_input_skips_readback() -> None:
# only secrets are read back.
el = _mock_input({"type": "text", "autocomplete": None, "name": "search"})
- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "not a secret value"
)
@@ -525,7 +410,7 @@ async def test_totp_value_short_circuits_before_secret_readback() -> None:
# `not is_totp_value` conjunct is a defensive backstop for this invariant.
el = _mock_input({"type": "password", "autocomplete": None, "name": "otp"})
- results, card_readback, tel_verify, phone_format, _, _, secret_readback = await _run_input_text(
+ results, card_readback, tel_verify, phone_format, _, secret_readback = await _run_input_text(
el, "{{ totp }}", resolved=str(BitwardenConstants.TOTP)
)
diff --git a/tests/unit/test_select_shadow_match.py b/tests/unit/test_select_shadow_match.py
index 2a644fd91..faed9f05e 100644
--- a/tests/unit/test_select_shadow_match.py
+++ b/tests/unit/test_select_shadow_match.py
@@ -1,4 +1,3 @@
-import copy
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, Mock
from zoneinfo import ZoneInfo
@@ -427,7 +426,6 @@ async def test_auto_completion_call_path_forwards_committed_value_to_shadow_matc
incremental_scraped = MagicMock(
start_listen_dom_increment=AsyncMock(),
stop_listen_dom_increment=AsyncMock(),
- get_incremental_elements_num=AsyncMock(return_value=1),
get_incremental_element_tree=AsyncMock(return_value=[_SHADOW_OPTION]),
)
incremental_scraped.build_html_tree.return_value = "<div>Committed Value</div>"
@@ -489,7 +487,9 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matc
click=AsyncMock(),
)
dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=selected_element))
- llm_handler = AsyncMock(return_value={"id": "matched-id", "value": " committed value ", "action_type": "click"})
+ llm_handler = AsyncMock(
+ return_value={"id": "different-id", "value": " committed value ", "action_type": "click"}
+ )
monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Committed Value</div>"))
monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))
@@ -526,622 +526,6 @@ async def test_emerging_select_call_path_forwards_committed_value_to_shadow_matc
assert events[0]["match_agrees_with_llm"] is True
-async def test_emerging_select_falls_back_to_visible_controlled_listbox_when_no_ids_are_new(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- anchor = {
- "id": "field-id",
- "tagName": "button",
- "attributes": {
- "aria-controls": "menu-dom-id",
- "aria-expanded": "true",
- "role": "combobox",
- },
- "interactable": True,
- "children": [],
- }
- listbox = {
- "id": "menu-id",
- "tagName": "div",
- "attributes": {"id": "menu-dom-id", "role": "listbox"},
- "children": [
- {
- "id": "option-a",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Alpha",
- "interactable": True,
- "children": [],
- },
- {
- "id": "option-hidden",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Beta",
- "interactable": False,
- "children": [],
- },
- {
- "id": "option-b",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Beta",
- "interactable": True,
- "children": [],
- },
- ],
- }
- tree = [anchor, listbox]
- ids = {
- "field-id": "[unique_id=field-id]",
- "menu-id": "[unique_id=menu-id]",
- "option-a": "[unique_id=option-a]",
- "option-hidden": "[unique_id=option-hidden]",
- "option-b": "[unique_id=option-b]",
- }
-
- def element_for_id(element_id: str) -> MagicMock:
- return MagicMock(
- is_interactable=Mock(return_value=element_id.startswith("option-") and element_id != "option-hidden")
- )
-
- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(side_effect=element_for_id))
-
- async def select_deterministically(**kwargs: object) -> tuple[handler.ActionSuccess, str]:
- get_option_candidates = kwargs["get_option_candidates"]
- assert callable(get_option_candidates)
- assert [candidate["label"] for candidate in get_option_candidates()] == ["Alpha", "Beta"]
- return handler.ActionSuccess(), "Beta"
-
- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
- deterministic_select = AsyncMock(side_effect=select_deterministically)
- monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
- llm_handler = AsyncMock()
- monkeypatch.setattr(
- handler.LLMAPIHandlerFactory,
- "get_override_llm_api_handler",
- Mock(return_value=llm_handler),
- )
- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
-
- scraped_page = SimpleNamespace(id_to_css_dict=ids)
- scraped_page_after_open = SimpleNamespace(id_to_css_dict=ids, element_tree=tree, element_tree_trimmed=tree)
- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
- result = await handler.select_from_emerging_elements(
- current_element_id="field-id",
- options=handler.CustomSelectPromptOptions(target_value="Beta"),
- page=MagicMock(),
- scraped_page=scraped_page,
- scraped_page_after_open=scraped_page_after_open,
- step=SimpleNamespace(step_id="step-1"),
- task=_shadow_task(),
- )
-
- assert isinstance(result, handler.ActionSuccess)
- deterministic_select.assert_awaited_once()
- llm_handler.assert_not_awaited()
-
-
-async def test_emerging_select_fallback_resolves_owned_listbox_when_trim_strips_ownership_attrs(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # A custom combobox whose listbox is linked by aria-controls. trim_element_tree drops
- # aria-controls / aria-owns / the DOM id from the trimmed tree, so ownership must be resolved
- # from the untrimmed element_tree — resolving it from the trimmed tree leaves the fallback inert.
- anchor = {
- "id": "field-id",
- "tagName": "button",
- "attributes": {"aria-controls": "menu-dom-id", "aria-expanded": "true", "role": "combobox"},
- "interactable": True,
- "children": [],
- }
- listbox = {
- "id": "menu-id",
- "tagName": "div",
- "attributes": {"id": "menu-dom-id", "role": "listbox"},
- "children": [
- {
- "id": "option-a",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Alpha",
- "interactable": True,
- "children": [],
- },
- {
- "id": "option-b",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Beta",
- "interactable": True,
- "children": [],
- },
- ],
- }
- element_tree = [anchor, listbox]
- element_tree_trimmed = handler.trim_element_tree(copy.deepcopy(element_tree))
- # Guard the premise: trim really removes the ownership signal the fallback keys off.
- assert "aria-controls" not in (element_tree_trimmed[0].get("attributes") or {})
- assert "id" not in (element_tree_trimmed[1].get("attributes") or {})
-
- ids = {
- "field-id": "[unique_id=field-id]",
- "menu-id": "[unique_id=menu-id]",
- "option-a": "[unique_id=option-a]",
- "option-b": "[unique_id=option-b]",
- }
-
- def element_for_id(element_id: str) -> MagicMock:
- return MagicMock(is_interactable=Mock(return_value=element_id.startswith("option-")))
-
- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(side_effect=element_for_id))
-
- async def select_deterministically(**kwargs: object) -> tuple[handler.ActionSuccess, str]:
- get_option_candidates = kwargs["get_option_candidates"]
- assert callable(get_option_candidates)
- assert [candidate["label"] for candidate in get_option_candidates()] == ["Alpha", "Beta"]
- return handler.ActionSuccess(), "Beta"
-
- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
- deterministic_select = AsyncMock(side_effect=select_deterministically)
- monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
- monkeypatch.setattr(
- handler.LLMAPIHandlerFactory,
- "get_override_llm_api_handler",
- Mock(return_value=AsyncMock()),
- )
-
- scraped_page = SimpleNamespace(id_to_css_dict=ids)
- scraped_page_after_open = SimpleNamespace(
- id_to_css_dict=ids,
- element_tree=element_tree,
- element_tree_trimmed=element_tree_trimmed,
- )
- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
- result = await handler.select_from_emerging_elements(
- current_element_id="field-id",
- options=handler.CustomSelectPromptOptions(target_value="Beta"),
- page=MagicMock(),
- scraped_page=scraped_page,
- scraped_page_after_open=scraped_page_after_open,
- step=SimpleNamespace(step_id="step-1"),
- task=_shadow_task(),
- )
-
- assert isinstance(result, handler.ActionSuccess)
- deterministic_select.assert_awaited_once()
-
-
-async def test_emerging_select_keeps_off_list_candidate_for_deterministic_match_when_flag_off(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # Option nodes in custom widgets are often not flagged interactable. With the verify flag off we
- # must not pre-filter deterministic candidates to the interactable set (main's behavior) — dropping
- # them there would regress selections that used to succeed.
- tree = [
- {
- "id": "option-x",
- "tagName": "div",
- "attributes": {"role": "option"},
- "text": "Alpha",
- "interactable": True,
- "children": [],
- },
- {
- "id": "option-y",
- "tagName": "div",
- "attributes": {"role": "option"},
- "text": "Beta",
- "children": [],
- },
- ]
- monkeypatch.setattr(handler, "_is_verify_emerging_select_pick_enabled", AsyncMock(return_value=False))
- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock())
- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div role='option'>option</div>"))
- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
- llm_handler = AsyncMock()
- monkeypatch.setattr(
- handler.LLMAPIHandlerFactory,
- "get_override_llm_api_handler",
- Mock(return_value=llm_handler),
- )
- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
-
- async def select_deterministically(**kwargs: object) -> tuple[handler.ActionSuccess, str]:
- get_option_candidates = kwargs["get_option_candidates"]
- assert callable(get_option_candidates)
- assert [candidate["element_id"] for candidate in get_option_candidates()] == ["option-x", "option-y"]
- return handler.ActionSuccess(), "Beta"
-
- deterministic_select = AsyncMock(side_effect=select_deterministically)
- monkeypatch.setattr(handler, "_select_deterministic_custom_option", deterministic_select)
-
- scraped_page = SimpleNamespace(id_to_css_dict={})
- scraped_page_after_open = SimpleNamespace(
- id_to_css_dict={"option-x": "[unique_id=option-x]", "option-y": "[unique_id=option-y]"},
- element_tree_trimmed=tree,
- )
- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
- result = await handler.select_from_emerging_elements(
- current_element_id="field-id",
- options=handler.CustomSelectPromptOptions(target_value="Beta"),
- page=MagicMock(),
- scraped_page=scraped_page,
- scraped_page_after_open=scraped_page_after_open,
- new_interactable_element_ids=["option-x"],
- step=SimpleNamespace(step_id="step-1"),
- task=_shadow_task(),
- )
-
- assert isinstance(result, handler.ActionSuccess)
- deterministic_select.assert_awaited_once()
- llm_handler.assert_not_awaited()
-
-
[email protected](("anchor_expanded", "expected"), [(False, []), (True, ["listbox-id"])])
-def test_custom_select_fallback_requires_open_anchor_for_sole_listbox(
- anchor_expanded: bool,
- expected: list[str],
-) -> None:
- tree = [
- {
- "id": "field-id",
- "tagName": "button",
- "attributes": {"aria-expanded": str(anchor_expanded).lower()},
- "children": [],
- },
- {
- "id": "listbox-id",
- "tagName": "div",
- "attributes": {"role": "listbox"},
- "children": [
- {
- "id": "option-id",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Alpha",
- "children": [],
- }
- ],
- },
- ]
-
- result = handler._custom_select_fallback_subtrees(tree, "field-id")
-
- assert [element["id"] for element in result] == expected
-
-
-def test_custom_select_fallback_accepts_sole_listbox_for_expanded_combobox_ancestor() -> None:
- tree = [
- {
- "id": "combobox-id",
- "tagName": "div",
- "attributes": {"aria-expanded": "true", "role": "combobox"},
- "children": [
- {
- "id": "field-id",
- "tagName": "input",
- "attributes": {},
- "children": [],
- }
- ],
- },
- {
- "id": "listbox-id",
- "tagName": "div",
- "attributes": {"role": "listbox"},
- "children": [
- {
- "id": "option-id",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Alpha",
- "children": [],
- }
- ],
- },
- ]
-
- result = handler._custom_select_fallback_subtrees(tree, "field-id")
-
- assert [element["id"] for element in result] == ["listbox-id"]
-
-
-async def test_emerging_select_rejects_page_global_options_without_owned_listbox(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- tree = [
- {
- "id": "field-id",
- "tagName": "button",
- "attributes": {"aria-expanded": "true", "role": "combobox"},
- "children": [],
- },
- {
- "id": "unrelated-option",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Wrong field",
- "children": [],
- },
- ]
- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock())
- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
- scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})
- scraped_page_after_open = SimpleNamespace(
- id_to_css_dict={"field-id": "[unique_id=field-id]"},
- element_tree=tree,
- element_tree_trimmed=tree,
- )
-
- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
- with pytest.raises(handler.NoIncrementalElementFoundForCustomSelection):
- await handler.select_from_emerging_elements(
- current_element_id="field-id",
- options=handler.CustomSelectPromptOptions(target_value="Wrong field"),
- page=MagicMock(),
- scraped_page=scraped_page,
- scraped_page_after_open=scraped_page_after_open,
- step=SimpleNamespace(step_id="step-1"),
- task=_shadow_task(),
- )
-
-
-async def test_emerging_select_rejects_hidden_candidate_from_deterministic_and_llm_paths(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- tree = [
- {
- "id": "hidden-id",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Exact target",
- "children": [],
- },
- {
- "id": "visible-id",
- "tagName": "button",
- "attributes": {"role": "option"},
- "text": "Visible alternative",
- "children": [],
- },
- ]
- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock())
- monkeypatch.setattr(handler, "_is_verify_emerging_select_pick_enabled", AsyncMock(return_value=True))
- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<button role='option'>option</button>"))
- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
- llm_handler = AsyncMock(return_value={"id": "hidden-id", "value": "Exact target", "action_type": "click"})
- monkeypatch.setattr(
- handler.LLMAPIHandlerFactory,
- "get_override_llm_api_handler",
- Mock(return_value=llm_handler),
- )
- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
-
- async def select_deterministically(**kwargs: object) -> None:
- get_option_candidates = kwargs["get_option_candidates"]
- assert callable(get_option_candidates)
- assert [candidate["element_id"] for candidate in get_option_candidates()] == ["visible-id"]
- return None
-
- monkeypatch.setattr(
- handler,
- "_select_deterministic_custom_option",
- AsyncMock(side_effect=select_deterministically),
- )
- scraped_page = SimpleNamespace(id_to_css_dict={})
- scraped_page_after_open = SimpleNamespace(
- id_to_css_dict={
- "hidden-id": "[unique_id=hidden-id]",
- "visible-id": "[unique_id=visible-id]",
- },
- element_tree_trimmed=tree,
- )
-
- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
- with pytest.raises(handler.NoAvailableOptionFoundForCustomSelection):
- await handler.select_from_emerging_elements(
- current_element_id="field-id",
- options=handler.CustomSelectPromptOptions(target_value="Exact target"),
- page=MagicMock(),
- scraped_page=scraped_page,
- scraped_page_after_open=scraped_page_after_open,
- new_interactable_element_ids=["visible-id"],
- step=SimpleNamespace(step_id="step-1"),
- task=_shadow_task(),
- )
-
- dom_after_open.get_skyvern_element_by_id.assert_not_awaited()
-
-
-async def test_emerging_select_allows_input_text_on_anchor_element(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- anchor_input = MagicMock(
- scroll_into_view=AsyncMock(),
- get_tag_name=Mock(return_value="input"),
- get_locator=Mock(return_value=MagicMock()),
- is_readonly=AsyncMock(return_value=False),
- input_clear=AsyncMock(),
- input_sequentially=AsyncMock(),
- )
- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=anchor_input))
- llm_handler = AsyncMock(return_value={"id": "field-id", "value": "filter text", "action_type": "input_text"})
- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Option</div>"))
- monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))
- monkeypatch.setattr(handler, "get_input_value", AsyncMock(return_value=""))
- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
- monkeypatch.setattr(
- handler.LLMAPIHandlerFactory,
- "get_override_llm_api_handler",
- Mock(return_value=llm_handler),
- )
- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
-
- scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})
- scraped_page_after_open = SimpleNamespace(
- id_to_css_dict={
- "field-id": "[unique_id=field-id]",
- "option-x": "[unique_id=option-x]",
- },
- element_tree_trimmed=[
- {
- "id": "option-x",
- "tagName": "div",
- "attributes": {"role": "option"},
- "text": "Other Option",
- "interactable": True,
- }
- ],
- )
- task = SimpleNamespace(
- navigation_goal="goal",
- navigation_payload={},
- llm_key=None,
- workflow_run_id=None,
- organization_id=None,
- workflow_permanent_id=None,
- )
- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
- result = await handler.select_from_emerging_elements(
- current_element_id="field-id",
- options=handler.CustomSelectPromptOptions(target_value="filter text"),
- page=MagicMock(),
- scraped_page=scraped_page,
- scraped_page_after_open=scraped_page_after_open,
- new_interactable_element_ids=["option-x"],
- step=SimpleNamespace(step_id="step-1"),
- task=task,
- )
-
- assert isinstance(result, handler.ActionSuccess)
- anchor_input.input_clear.assert_awaited_once()
- anchor_input.input_sequentially.assert_awaited_once_with("filter text")
-
-
-async def test_emerging_select_rejects_input_text_on_element_outside_anchor_and_new_ids(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock())
- llm_handler = AsyncMock(return_value={"id": "stranger-id", "value": "filter text", "action_type": "input_text"})
- monkeypatch.setattr(handler, "_is_verify_emerging_select_pick_enabled", AsyncMock(return_value=True))
- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Option</div>"))
- monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))
- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
- monkeypatch.setattr(
- handler.LLMAPIHandlerFactory,
- "get_override_llm_api_handler",
- Mock(return_value=llm_handler),
- )
- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
-
- scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})
- scraped_page_after_open = SimpleNamespace(
- id_to_css_dict={
- "field-id": "[unique_id=field-id]",
- "option-x": "[unique_id=option-x]",
- },
- element_tree_trimmed=[
- {
- "id": "option-x",
- "tagName": "div",
- "attributes": {"role": "option"},
- "text": "Other Option",
- "interactable": True,
- }
- ],
- )
- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
- with pytest.raises(handler.NoAvailableOptionFoundForCustomSelection):
- await handler.select_from_emerging_elements(
- current_element_id="field-id",
- options=handler.CustomSelectPromptOptions(target_value="filter text"),
- page=MagicMock(),
- scraped_page=scraped_page,
- scraped_page_after_open=scraped_page_after_open,
- new_interactable_element_ids=["option-x"],
- step=SimpleNamespace(step_id="step-1"),
- task=_shadow_task(),
- )
-
- dom_after_open.get_skyvern_element_by_id.assert_not_awaited()
-
-
-async def test_emerging_select_off_list_pick_still_proceeds_when_flag_disabled(
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- # Flag off keeps today's behavior: an off-list pick is still acted on rather than rejected, so
- # merging this cannot change any org's outcomes until the flag is ramped.
- selected_element = MagicMock(
- get_tag_name=Mock(return_value="div"),
- get_attr=AsyncMock(return_value=None),
- scroll_into_view=AsyncMock(),
- click=AsyncMock(),
- )
- dom_after_open = MagicMock(get_skyvern_element_by_id=AsyncMock(return_value=selected_element))
- llm_handler = AsyncMock(return_value={"id": "stranger-id", "value": None, "action_type": "click"})
- monkeypatch.setattr(handler, "_is_verify_emerging_select_pick_enabled", AsyncMock(return_value=False))
- monkeypatch.setattr(handler, "DomUtil", Mock(return_value=dom_after_open))
- monkeypatch.setattr(handler, "json_to_html", Mock(return_value="<div>Option</div>"))
- monkeypatch.setattr(handler, "_select_deterministic_custom_option", AsyncMock(return_value=None))
- monkeypatch.setattr(handler.prompt_engine, "load_prompt", Mock(return_value="prompt"))
- monkeypatch.setattr(
- handler.LLMAPIHandlerFactory,
- "get_override_llm_api_handler",
- Mock(return_value=llm_handler),
- )
- monkeypatch.setattr(handler.app, "CUSTOM_SELECT_AGENT_LLM_API_HANDLER", llm_handler)
-
- scraped_page = SimpleNamespace(id_to_css_dict={"field-id": "[unique_id=field-id]"})
- scraped_page_after_open = SimpleNamespace(
- id_to_css_dict={
- "field-id": "[unique_id=field-id]",
- "option-x": "[unique_id=option-x]",
- },
- element_tree_trimmed=[
- {
- "id": "option-x",
- "tagName": "div",
- "attributes": {"role": "option"},
- "text": "Other Option",
- "interactable": True,
- }
- ],
- )
- with skyvern_context.scoped(SkyvernContext(tz_info=ZoneInfo("UTC"))):
- with structlog.testing.capture_logs() as logs:
- result = await handler.select_from_emerging_elements(
- current_element_id="field-id",
- options=handler.CustomSelectPromptOptions(target_value="filter text"),
- page=MagicMock(),
- scraped_page=scraped_page,
- scraped_page_after_open=scraped_page_after_open,
- new_interactable_element_ids=["option-x"],
- step=SimpleNamespace(step_id="step-1"),
- task=_shadow_task(),
- )
-
- assert isinstance(result, handler.ActionSuccess)
- selected_element.click.assert_awaited()
- # Flag off still emits telemetry so the reject rate is measurable before ramping the flag on.
- off_list_warnings = [
- log for log in logs if log.get("log_level") == "warning" and log.get("element_id") == "stranger-id"
- ]
- assert off_list_warnings
-
-
@pytest.mark.asyncio
async def test_dropdown_select_call_path_forwards_committed_value_to_shadow_match(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_tel_input_bare_digits.py b/tests/unit/test_tel_input_bare_digits.py
index bac4d4da1..463800368 100644
--- a/tests/unit/test_tel_input_bare_digits.py
+++ b/tests/unit/test_tel_input_bare_digits.py
@@ -8,7 +8,6 @@ import pytest
from skyvern.webeye.actions import handler_utils
from skyvern.webeye.actions.handler import (
- _is_org_flag_enabled,
_is_tel_digit_fix_enabled,
_nanp_e164_fallback,
_nanp_national_digits,
@@ -187,29 +186,3 @@ async def test_is_tel_digit_fix_enabled_uses_org_keyed_rollout(monkeypatch: pyte
organization.organization_id,
properties={"organization_id": organization.organization_id},
)
-
-
-async def test_is_org_flag_enabled_buckets_any_flag_by_org(monkeypatch: pytest.MonkeyPatch) -> None:
- now = datetime.now(UTC)
- organization = make_organization(now)
- task = make_task(now, organization)
- provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(return_value=True))
- monkeypatch.setattr("skyvern.webeye.actions.handler.app.EXPERIMENTATION_PROVIDER", provider)
-
- assert await _is_org_flag_enabled(task, "SOME_OTHER_FLAG", "some-other") is True
- provider.is_feature_enabled_cached.assert_awaited_once_with(
- "SOME_OTHER_FLAG",
- organization.organization_id,
- properties={"organization_id": organization.organization_id},
- )
-
-
-async def test_is_org_flag_enabled_fails_closed_when_provider_raises(monkeypatch: pytest.MonkeyPatch) -> None:
- now = datetime.now(UTC)
- task = make_task(now, make_organization(now))
- provider = SimpleNamespace(is_feature_enabled_cached=AsyncMock(side_effect=RuntimeError("posthog down")))
- monkeypatch.setattr("skyvern.webeye.actions.handler.app.EXPERIMENTATION_PROVIDER", provider)
-
- assert await _is_org_flag_enabled(task, "SOME_OTHER_FLAG", "some-other") is False