Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Bound the engine 1.0 element lookup so a frozen page cannot stall an action (#8990)
Aaron Perez73e49ce7bb3d
diff --git a/skyvern/config.py b/skyvern/config.pyindex b57d85286..940d39abb 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -141,6 +141,11 @@ class Settings(BaseSettings):TEMP_PATH: str = "./temp"DOWNLOAD_PATH: str = f"{REPO_ROOT_DIR}/downloads"BROWSER_ACTION_TIMEOUT_MS: int = 5000+ # Playwright's `Locator.count()` and the iframe-chain `query_selector`/`content_frame` probes take+ # no timeout argument, so a renderer that stopped answering leaves each one waiting on a protocol+ # reply forever. The action deadline adds its headroom to this budget; expiry means the page stopped+ # answering, not that a healthy lookup was merely slow.+ BROWSER_ELEMENT_LOOKUP_TIMEOUT_MS: int = 5000BROWSER_ACTION_MAX_EXECUTION_SECONDS: int = 1200WORKER_STALL_DUMP_SECONDS: int = 0# "enforce" is deliberately absent: the policy core can only observe until every sink consumes@@ -181,6 +186,12 @@ class Settings(BaseSettings):# picks from whatever loaded — so this is kept short enough that a never-settling menu costs# seconds rather than the minutes a run's elapsed budget cannot absorb.OPTION_LOADING_TIMEOUT_MS: int = 60000+ # Wall-clock budget for the autocomplete guessed-value fanout, measured from the ladder's first+ # probe. The probe for the requested value always runs; this only gates the LLM-guessed retries,+ # each of which costs a full clear-and-retype plus a DOM scrape. Without it a control whose+ # options never match spent minutes per field before the discovery fallback was reached. The budget+ # stops new guesses from starting; a guess already in progress may overrun it. 0 disables this limit.+ AUTO_COMPLETION_LADDER_BUDGET_SECONDS: float = 60.0MAX_STEPS_PER_RUN: int = 10MAX_STEPS_PER_TASK_V2: int = 25MAX_ITERATIONS_PER_TASK_V2: int = 50diff --git a/skyvern/webeye/action_deadline.py b/skyvern/webeye/action_deadline.pyindex e29d0083c..4b5db0c6e 100644--- a/skyvern/webeye/action_deadline.py+++ b/skyvern/webeye/action_deadline.py@@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Awaitable, Callablefrom contextlib import asynccontextmanagerfrom typing import Any, TypeVar+from skyvern.config import settingsfrom skyvern.exceptions import ActionDeadlineExceededfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.webeye.browser_health import BrowserOperation@@ -18,6 +19,15 @@ T = TypeVar("T")_UNSET = object()+async def bounded_lookup_call(call: Callable[[], Awaitable[T]]) -> T:+ """Bound a lookup call without a native Playwright timeout, such as ``Locator.count()``."""+ async with under_action_deadline(+ budget_ms=settings.BROWSER_ELEMENT_LOOKUP_TIMEOUT_MS,+ operation=BrowserOperation.EVALUATE,+ ):+ return await cancellable_driver_call(call)++def cancellation_pending() -> bool:"""Whether this task carries an unhandled cancellation request.diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex faa8b4c83..34493cf9b 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -274,6 +274,7 @@ from skyvern.webeye.utils.dom import (InteractiveElement,SkyvernElement,SkyvernOptionType,+ bounded_lookup_call,is_element_detached_error,is_incompatible_text_input_error,is_post_dispatch_click_timeout,@@ -2655,7 +2656,7 @@ class CustomSingleSelectResult:if not isinstance(self.action_result, ActionSuccess):return True- if await self.dropdown_menu.get_locator().count() == 0:+ if await bounded_lookup_call(self.dropdown_menu.get_locator().count) == 0:return Truereturn not await self.skyvern_frame.get_element_visible(self.dropdown_menu.get_locator())@@ -6435,8 +6436,10 @@ async def _batched_target_stale_beyond_remap(return False # only the main frame has a scrape-stable identity to reason abouttry:locator, frame_content = await resolve_locator(scraped_page, page, frame, css)- if await locator.count() == 1:+ if await bounded_lookup_call(locator.count) == 1:return False # the exact injected node is still live -> not stale+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:return False # cannot confirm staleness -> decline; legacy dispatch stays authoritativereturn True@@ -6497,8 +6500,10 @@ async def _refresh_stale_web_action_before_dispatch(return Nonetry:locator, frame_content = await resolve_locator(scraped_page, page, frame, css)- if await locator.count() == 1:+ if await bounded_lookup_call(locator.count) == 1:return None # the exact injected node is still live -> not stale -> dispatch unchanged+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:return None # cannot confirm liveness / marker survival -> decline; legacy path authoritative@@ -6643,7 +6648,7 @@ async def _is_single_select_option_highlight(element: SkyvernElement) -> bool:if (await element.get_attr("role", mode="dynamic") or "").strip().casefold() != "option":return Falsemultiselectable = element.get_locator().locator("xpath=ancestor-or-self::*[@aria-multiselectable][1]")- if await multiselectable.count() == 0:+ if await bounded_lookup_call(multiselectable.count) == 0:return Truereturn (await multiselectable.get_attribute("aria-multiselectable") or "").strip().casefold() != "true"@@ -6698,6 +6703,8 @@ async def _resolve_static_listbox_container_target(if target is None or not await _is_static_listbox_option(target) or await target.is_disabled(dynamic=True):return _static_listbox_replan_failure(action, "The static listbox option is unavailable; rescrape it.")return target+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:LOG.debug("Static listbox container inspection failed", element_id=element.get_id())return None@@ -6991,7 +6998,7 @@ async def _drive_grid_row_selection(click does not achieve the desired selection. Never loops, force-clicks, or mutates the DOM."""cell_locator = element.get_locator().locator('xpath=ancestor-or-self::*[@role="gridcell" or self::td][1]').firsttry:- if await cell_locator.count() == 0:+ if await bounded_lookup_call(cell_locator.count) == 0:return Noneif await SkyvernElement._label_click_forwards_to_descendant(cell_locator, fail_closed=True):LOG.warning(@@ -7007,6 +7014,8 @@ async def _drive_grid_row_selection()return Noneawait cell_locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS, position=point)+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:LOG.warning("Grid row-selection click failed, continuing the normal click", element_id=element.get_id(), exc_info=True@@ -7080,6 +7089,8 @@ async def _resolve_live_click_state(LOG.info("Trusting aria-selected on a static listbox option", element_id=element.get_id())return _EffectiveClickState(checked=parsed, control_type=None)return _EffectiveClickState(checked=None, control_type=None)+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:LOG.debug("Failed to read live selected state; continuing with an ordinary click", element_id=element.get_id())return _EffectiveClickState(checked=None, control_type=None)@@ -7091,7 +7102,7 @@ async def _resolve_live_selected_state(element: SkyvernElement) -> bool | None:async def _get_associated_checkbox_label_locator(element: SkyvernElement) -> Locator | None:ancestor_label_locator = element.get_locator().locator("xpath=ancestor::label[1]")- if await ancestor_label_locator.count() > 0:+ if await bounded_lookup_call(ancestor_label_locator.count) > 0:return ancestor_label_locatorinput_id = await element.get_attr("id", mode="dynamic")@@ -7099,7 +7110,7 @@ async def _get_associated_checkbox_label_locator(element: SkyvernElement) -> Locreturn Noneexplicit_label_locator = element.get_frame().locator(f"label[for={json.dumps(input_id)}]")- if await explicit_label_locator.count() > 0:+ if await bounded_lookup_call(explicit_label_locator.count) > 0:return explicit_label_locator.firstreturn None@@ -9080,6 +9091,59 @@ def _emit_tel_input_outcome(action.tel_input_outcome = None+async def _press_autocomplete_commit_tab(*, skyvern_element: SkyvernElement, action: actions.Action) -> None:+ if not await skyvern_element.is_visible() or await skyvern_element.is_raw_input() or action.skip_auto_complete_tab:+ return+ LOG.debug(+ "Trigger input-selection hack, pressing Tab to choose one",+ action=action,+ )+ await skyvern_element.press_key("Tab")+++async def _autocomplete_left_field_empty(+ *,+ skyvern_element: SkyvernElement,+ tag_name: str,+ engine_selection: BrowserEngineSelection | None,+ expected_value: str,+ is_secret_value: bool,+) -> bool:+ try:+ value = await get_input_value(+ tag_name=tag_name,+ locator=skyvern_element.get_locator(),+ engine_selection=engine_selection,+ # This runs on the already-slow exhausted-ladder path, so it must not inherit+ # Playwright's 30s default wait on a control the commit may have detached.+ read_timeout_ms=settings.BROWSER_ACTION_TIMEOUT_MS,+ )+ except Exception:+ # A node that cannot be read -- detached, renavigated, or type-incompatible -- is no evidence+ # of an empty field, and treating it as one would fail actions whose value did land.+ LOG.info(+ "Could not read the auto-completion field back, not treating it as empty",+ element_id=skyvern_element.get_id(),+ exc_info=True,+ )+ return False+ if value is None:+ return False+ if value.strip():+ return False++ committed, _ = await _custom_select_scope_confirms_committed(+ readback_scope_element=skyvern_element,+ anchor_is_combobox_input=tag_name == InteractiveElement.INPUT,+ matched_element_id=skyvern_element.get_id(),+ matched_label="<secret>" if is_secret_value else expected_value,+ expected_label=_normalize_select_shadow_text(expected_value),+ allow_aria_selected_option_tokens=False,+ allow_single_value_scope=tag_name == InteractiveElement.INPUT,+ )+ return not committed++@traced(name="skyvern.agent.action.input_text")async def handle_input_text_action(action: actions.InputTextAction,@@ -9367,6 +9431,9 @@ async def _handle_input_text_action(incremental_element: list[dict] = []auto_complete_hacky_flag: bool = False+ # Set when the auto-completion ladder ran every attempt and committed no option, so the plain+ # fill below is typing text that matched nothing in the control's list.+ autocomplete_ladder_unresolved: bool = False# Set when the typeahead prefilter (below) types the target into the field. If that pre-input# selection does not commit, the terminal fill must clear first so the typed-but-uncommitted value# is not doubled. `prefilter_attempted` stays set even when the typing raises mid-dispatch (leaving a@@ -9597,9 +9664,9 @@ async def _handle_input_text_action("Find a blocking element to the current element, going to blur the blocking element first",blocking_element=blocking_element.get_locator(),)- if await blocking_element.get_locator().count():+ if await bounded_lookup_call(blocking_element.get_locator().count):await blocking_element.press_key("Escape")- if await blocking_element.get_locator().count():+ if await bounded_lookup_call(blocking_element.get_locator().count):await blocking_element.blur()if try_to_quit_dropdown and await skyvern_element.is_visible():@@ -9977,6 +10044,9 @@ async def _handle_input_text_action():auto_complete_hacky_flag = Falsereturn [result]+ # A search bar legitimately keeps free text with nothing selected, so only a+ # non-search-bar control can have been left blank by the exhausted ladder.+ autocomplete_ladder_unresolved = not input_or_select_context.is_search_bar# Only the bare-digit NANP path uses the verified fallback ladder; other tel shapes remain# observational to preserve flag-off behavior.@@ -10201,7 +10271,7 @@ async def _handle_input_text_action(if restoredelse "its previous text could not be put back"),- target_value=text,+ target_value="<secret>" if is_secret_value else text,))]@@ -10349,6 +10419,34 @@ async def _handle_input_text_action(# Always stop listeningawait incremental_scraped.stop_listen_dom_increment()+ if autocomplete_ladder_unresolved:+ # Commit here rather than in the outer finally: the Tab is what blurs the control, and an+ # autocomplete that rejects unmatched text clears itself on blur. Only a read-back taken+ # after the commit can confirm that a matching value was committed.+ if auto_complete_hacky_flag:+ auto_complete_hacky_flag = False+ await _press_autocomplete_commit_tab(skyvern_element=skyvern_element, action=action)+ if await _autocomplete_left_field_empty(+ skyvern_element=skyvern_element,+ tag_name=tag_name,+ engine_selection=engine_selection,+ expected_value=text,+ is_secret_value=is_secret_value,+ ):+ LOG.info(+ "Auto completion committed no matching option, reporting a failed action",+ element_id=skyvern_element.get_id(),+ )+ failure = ActionFailure(+ NoSuitableAutoCompleteOption(+ reasoning="the auto-completion options matched no attempted value and no matching option was committed",+ target_value="<secret>" if is_secret_value else text,+ )+ )+ # Prevent a duplicate-element action such as Submit from following this failure.+ failure.skip_remaining_actions = True+ return [failure]+return [ActionSuccess()]except (SkyvernPageAnalysisTimeout, InvalidElementForTextInput):raise@@ -10375,17 +10473,8 @@ async def _handle_input_text_action(tel_outcome.browser_valid = await _probe_tel_browser_validity(skyvern_element.get_locator())# HACK: force to finish missing auto completion input- if (- auto_complete_hacky_flag- and await skyvern_element.is_visible()- and not await skyvern_element.is_raw_input()- and not action.skip_auto_complete_tab- ):- LOG.debug(- "Trigger input-selection hack, pressing Tab to choose one",- action=action,- )- await skyvern_element.press_key("Tab")+ if auto_complete_hacky_flag:+ await _press_autocomplete_commit_tab(skyvern_element=skyvern_element, action=action)_URL_RECOVERY_EDIT_DISTANCE_FRACTION = 0.1@@ -12565,7 +12654,7 @@ async def choose_auto_completion_dropdown(if not element_id:continuelocator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')- cnt = await locator.count()+ cnt = await bounded_lookup_call(locator.count)if cnt == 0:continue@@ -12605,7 +12694,7 @@ async def choose_auto_completion_dropdown(matched_element_id = matched_candidate.get("element_id") or ""matched_label = matched_candidate.get("label") or ""matched_locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{matched_element_id}"]')- if matched_element_id and matched_label and await matched_locator.count() > 0:+ if matched_element_id and matched_label and await bounded_lookup_call(matched_locator.count) > 0:option_identity_matches = await _verify_autocomplete_option_identity(skyvern_frame=skyvern_frame,locator=matched_locator,@@ -12734,7 +12823,7 @@ async def choose_auto_completion_dropdown(input_normalized = " ".join(text.lower().split())if fast_path_element_id and input_normalized and input_normalized in option_text:fast_path_locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{fast_path_element_id}"]')- if await fast_path_locator.count() > 0:+ if await bounded_lookup_call(fast_path_locator.count) > 0:LOG.info("Location auto-completion fast path: single option found, skipping LLM",element_id=fast_path_element_id,@@ -12824,7 +12913,10 @@ async def choose_auto_completion_dropdown(if not element_id:reasoning = json_response.get("reasoning")- raise NoSuitableAutoCompleteOption(reasoning=reasoning, target_value=text)+ raise NoSuitableAutoCompleteOption(+ reasoning=reasoning,+ target_value="<secret>" if is_secret_value else text,+ )if relevance_float < relevance_threshold:LOG.info(@@ -12846,7 +12938,7 @@ async def choose_auto_completion_dropdown()locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')- if await locator.count() == 0:+ if await bounded_lookup_call(locator.count) == 0:raise MissingElement(element_id=element_id)# Use SkyvernElement.click() so we get the full fallback chain@@ -12876,6 +12968,8 @@ async def choose_auto_completion_dropdown(clear_input = Falsereturn result+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception as e:LOG.info("Failed to choose the auto completion dropdown",@@ -12927,6 +13021,7 @@ async def input_or_auto_complete_input(collapse_autocomplete_fanout_enabled: bool = False,*,is_secret_value: bool,+ monotonic: Callable[[], float] = time.monotonic,) -> ActionResult | None:action_phase.mark_action_phase(ActionPhase.AUTOCOMPLETE)LOG.info(@@ -12945,6 +13040,26 @@ async def input_or_auto_complete_input(current_value = textresult = AutoCompletionResult()+ ladder_started_at = monotonic()+ retry_budget_seconds = settings.AUTO_COMPLETION_LADDER_BUDGET_SECONDS+ # Per-attempt value and duration. An attempt is silent for its whole runtime -- no log, no span --+ # so without this a ladder that spends minutes is indistinguishable from one that spends seconds.+ attempt_timings: list[dict[str, str | float]] = []++ def _elapsed_seconds() -> float:+ return monotonic() - ladder_started_at++ def _retry_budget_exhausted() -> bool:+ return retry_budget_seconds > 0 and _elapsed_seconds() >= retry_budget_seconds++ def _record_attempt(value: str, started_at: float) -> None:+ attempt_timings.append(+ {+ "value": "<secret>" if is_secret_value else value,+ "seconds": round(monotonic() - started_at, 1),+ }+ )+while current_attemp < MAX_AUTO_COMPLETE_ATTEMP:current_attemp += 1whole_new_elements: list[dict] = []@@ -12956,6 +13071,7 @@ async def input_or_auto_complete_input(input_value=current_value,)is_location = input_or_select_context.is_location_input or False+ attempt_started_at = monotonic()result = await choose_auto_completion_dropdown(context=input_or_select_context,page=page,@@ -12971,6 +13087,7 @@ async def input_or_auto_complete_input(action=action,is_secret_value=is_secret_value,)+ _record_attempt(current_value, attempt_started_at)if isinstance(result.action_result, ActionSuccess):return result.action_result@@ -12995,29 +13112,40 @@ async def input_or_auto_complete_input()values: list[dict] = []- prompt = prompt_engine.load_prompt(- "auto-completion-potential-answers",- potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,- field_information=field_information,- current_value=current_value,- navigation_goal=task.navigation_goal,- navigation_payload_str=json.dumps(task.navigation_payload),- local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),- )+ if _retry_budget_exhausted():+ # Guesses the ladder cannot afford to type are not worth asking an LLM for; the+ # discovery fallback below opens the unfiltered control instead.+ LOG.info(+ "Auto completion retry budget already spent by the first probe, skipping the potential-value fanout",+ element_id=skyvern_element.get_id(),+ elapsed_seconds=round(_elapsed_seconds(), 1),+ budget_seconds=retry_budget_seconds,+ attempts=attempt_timings,+ )+ else:+ prompt = prompt_engine.load_prompt(+ "auto-completion-potential-answers",+ potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,+ field_information=field_information,+ current_value=current_value,+ navigation_goal=task.navigation_goal,+ navigation_payload_str=json.dumps(task.navigation_payload),+ local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),+ )- LOG.info(- "Ask LLM to give potential values based on the current value",- current_value=current_value,- potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,- )- if collapse_autocomplete_fanout_enabled and action is not None:- action.set_has_mini_agent()- json_respone = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(- prompt=prompt, step=step, prompt_name="auto-completion-potential-answers"- )- values = json_respone.get("potential_values", [])+ LOG.info(+ "Ask LLM to give potential values based on the current value",+ current_value=current_value,+ potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,+ )+ if collapse_autocomplete_fanout_enabled and action is not None:+ action.set_has_mini_agent()+ json_respone = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(+ prompt=prompt, step=step, prompt_name="auto-completion-potential-answers"+ )+ values = json_respone.get("potential_values", [])- for each_value in values:+ for value_index, each_value in enumerate(values):value: str = each_value.get("value", "")if not value:LOG.info(@@ -13025,11 +13153,22 @@ async def input_or_auto_complete_input(value=each_value,)continue+ if _retry_budget_exhausted():+ LOG.info(+ "Auto completion retry budget exhausted, skipping the remaining potential values",+ element_id=skyvern_element.get_id(),+ elapsed_seconds=round(_elapsed_seconds(), 1),+ budget_seconds=retry_budget_seconds,+ attempts=attempt_timings,+ skipped_value_count=len(values) - value_index,+ )+ breakLOG.info("Try the potential value for auto completion",sampling=True,input_value=value,)+ attempt_started_at = monotonic()result = await choose_auto_completion_dropdown(context=input_or_select_context,page=page,@@ -13046,6 +13185,7 @@ async def input_or_auto_complete_input(allow_full_page_rescrape=not initial_probe_found_no_options,is_secret_value=is_secret_value,)+ _record_attempt(value, attempt_started_at)if isinstance(result.action_result, ActionSuccess):return result.action_result@@ -13109,6 +13249,8 @@ async def input_or_auto_complete_input("Auto completion didn't finish, this might leave the input value to be empty.",sampling=True,context=input_or_select_context,+ attempts=attempt_timings,+ total_seconds=round(_elapsed_seconds(), 1),)return None@@ -14006,9 +14148,11 @@ async def _evaluate_element_scoped(async def _read_custom_select_matched_state(element: SkyvernElement) -> dict | None:try:- if await element.get_locator().count() != 1:+ if await bounded_lookup_call(element.get_locator().count) != 1:return Nonestate = await _evaluate_element_scoped(element, _CUSTOM_SELECT_MATCHED_STATE_JS)+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:LOG.info("Failed to read custom-select matched element state",diff --git a/skyvern/webeye/actions/multi_field_totp.py b/skyvern/webeye/actions/multi_field_totp.pyindex 5b2d83344..cd082302c 100644--- a/skyvern/webeye/actions/multi_field_totp.py+++ b/skyvern/webeye/actions/multi_field_totp.py@@ -15,13 +15,14 @@ import structlogfrom playwright.async_api import Locator, Pagefrom skyvern.constants import SKYVERN_ID_ATTR+from skyvern.exceptions import SkyvernPageAnalysisTimeoutfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.core.skyvern_context import MultiFieldTotpAttemptfrom skyvern.utils.contained_effects import contained_effectfrom skyvern.webeye.scraper.scraped_page import ScrapedPagefrom skyvern.webeye.scraper.scraper import structural_identityfrom skyvern.webeye.utils.document import get_main_document_loader_id-from skyvern.webeye.utils.dom import resolve_locator+from skyvern.webeye.utils.dom import bounded_lookup_call, resolve_locatorfrom skyvern.webeye.utils.page import SkyvernFrameLOG = structlog.get_logger()@@ -499,7 +500,7 @@ async def _refresh_multi_field_totp_group_binding(if not css or not frame:return classified("box_locator_metadata_missing", MultiFieldTotpBindingFailure.UNCONFIRMED)locator, _ = await resolve_locator(scraped_page, page, frame, css)- count = await locator.count()+ count = await bounded_lookup_call(locator.count)if count not in (0, 1):return classified("original_box_count_ambiguous", MultiFieldTotpBindingFailure.UNCONFIRMED)live_boxes.append(count == 1)@@ -523,7 +524,7 @@ async def _refresh_multi_field_totp_group_binding(if not container_css or not container_frame or container_frame != original_group.frame:return classified("container_locator_metadata_missing", MultiFieldTotpBindingFailure.UNCONFIRMED)locator, _ = await resolve_locator(scraped_page, page, container_frame, container_css)- container_count = await locator.count()+ container_count = await bounded_lookup_call(locator.count)if container_count not in (0, 1):return classified("original_container_count_ambiguous", MultiFieldTotpBindingFailure.UNCONFIRMED)container_live = container_count == 1@@ -585,6 +586,8 @@ async def _refresh_multi_field_totp_group_binding(if not loader_id_known:return classified("loader_id_unknown", MultiFieldTotpBindingFailure.UNCONFIRMED)return fresh, group.box_element_ids+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:return classified("observation_failed", MultiFieldTotpBindingFailure.UNCONFIRMED)@@ -712,7 +715,7 @@ class MultiFieldTotpFillObserver:async def refresh(self, boxes: list[Locator]) -> None:try:- if not boxes or await boxes[0].count() != 1:+ if not boxes or await bounded_lookup_call(boxes[0].count) != 1:returncount = await boxes[0].evaluate("(box, prefix) => box.ownerDocument.__skyvernOtpFillObserver?.prefix === prefix ? Number(box.ownerDocument.documentElement.getAttribute('data-skyvern-otp-max-filled')) : 0",@@ -721,6 +724,8 @@ class MultiFieldTotpFillObserver:)if isinstance(count, int) and not isinstance(count, bool) and 0 <= count <= self.attempt.expected_digits:self.attempt.observed_max_filled = max(self.attempt.observed_max_filled, count)+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:# The console count survives destruction of the document that held the attribute.pass@@ -782,13 +787,15 @@ async def _multi_field_totp_widget_scopes(boxes: list[Locator], *, include_ancesif not boxes:return []form = boxes[0].locator("xpath=ancestor::form[1]")- if await form.count() == 1 and all([await form.filter(has=box).count() == 1 for box in boxes]):+ if await bounded_lookup_call(form.count) == 1 and all(+ [await bounded_lookup_call(form.filter(has=box).count) == 1 for box in boxes]+ ):return [form]scopes = []ancestors = boxes[0].locator("xpath=ancestor::*[not(self::body or self::html)]")- for index in reversed(range(await ancestors.count())):+ for index in reversed(range(await bounded_lookup_call(ancestors.count))):ancestor = ancestors.nth(index)- if all([await ancestor.filter(has=box).count() == 1 for box in boxes]):+ if all([await bounded_lookup_call(ancestor.filter(has=box).count) == 1 for box in boxes]):scopes.append(ancestor)if not include_ancestors or len(scopes) == 3:break@@ -992,7 +999,7 @@ async def multi_field_totp_submission_evidence(boxes: list[Locator] = []try:boxes = await _retry_box_locators(page, scraped_page, attempt.box_element_ids)- counts = [await box.count() for box in boxes]+ counts = [await bounded_lookup_call(box.count) for box in boxes]if not all(count == 1 for count in counts):binding = await _refresh_multi_field_totp_group_binding(scraped_page, page, attempt)if isinstance(binding, MultiFieldTotpBindingFailure):@@ -1034,7 +1041,9 @@ async def _multi_field_totp_document_changed(page: Page, original_url: str, old_async def _multi_field_totp_boxes_disappeared(boxes: list[Locator]) -> bool:try:- return bool(boxes) and all([await box.count() == 0 for box in boxes])+ return bool(boxes) and all([await bounded_lookup_call(box.count) == 0 for box in boxes])+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:return False@@ -1155,7 +1164,7 @@ async def submit_multi_field_totp_retry(candidates_considered += 1control, fields = candidate.locator, candidate.log_fieldstry:- if await control.count() != 1:+ if await bounded_lookup_call(control.count) != 1:record("trial", "failed", fields, reason="submit_control_not_actionable")continueawait dispatch(control, "trial", fields)@@ -1164,9 +1173,11 @@ async def submit_multi_field_totp_retry(with timed("pre_evidence_ms"):if await multi_field_totp_submission_evidence(page, scraped_page, attempt):return True- if await control.count() != 1:+ if await bounded_lookup_call(control.count) != 1:record("click", "failed", fields, reason="submit_control_not_actionable")continue+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception as exc:last_trial_error = exccontinue@@ -1174,7 +1185,7 @@ async def submit_multi_field_totp_retry(breakphase = "click" if chosen is not None else "enter"if chosen is None:- if not boxes or await boxes[-1].count() != 1:+ if not boxes or await bounded_lookup_call(boxes[-1].count) != 1:record("enter", "failed", {}, reason="submit_control_not_actionable")raise MultiFieldTotpSubmitControlNotActionable() from last_trial_errorchosen = boxes[-1]@@ -1184,7 +1195,7 @@ async def submit_multi_field_totp_retry(with timed("pre_evidence_ms"):if await multi_field_totp_submission_evidence(page, scraped_page, attempt):return True- if await chosen.count() != 1:+ if await bounded_lookup_call(chosen.count) != 1:record("enter", "failed", fields, reason="submit_control_not_actionable")raise MultiFieldTotpSubmitControlNotActionable() from last_trial_errortry:diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex c60fd9de5..9503e3628 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -36,6 +36,7 @@ from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_spanfrom skyvern.utils.image_resizer import Resolutionfrom skyvern.utils.token_counter import approx_count_tokensfrom skyvern.utils.url_validators import strip_query_params+from skyvern.webeye.action_deadline import bounded_lookup_callfrom skyvern.webeye.browser_state import BLANK_PAGE_URLS, BrowserStatefrom skyvern.webeye.scraper.scraped_page import (CleanupElementTreeFunc,@@ -1154,7 +1155,7 @@ class IncrementalScrapePage(ElementTreeBuilder):if is_match:if child_locator:return child_locator, True- if interactable and parent_locator and await parent_locator.count() > 0:+ if interactable and parent_locator and await bounded_lookup_call(parent_locator.count) > 0:return parent_locator, Truereturn None, True@@ -1165,7 +1166,7 @@ class IncrementalScrapePage(ElementTreeBuilder):if text != value:return None, False- if await parent_locator.count() == 0:+ if await bounded_lookup_call(parent_locator.count) == 0:return None, Falseif not interactable:diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.pyindex b0f080409..a69102af2 100644--- a/skyvern/webeye/utils/dom.py+++ b/skyvern/webeye/utils/dom.py@@ -33,10 +33,12 @@ from skyvern.exceptions import (NoElementBoudingBox,NoneFrameError,SkyvernException,+ SkyvernPageAnalysisTimeout,)from skyvern.experimentation.wait_utils import get_or_create_wait_config, get_wait_time, scroll_into_view_waitfrom skyvern.forge.sdk.event.factory import EventStrategyFactoryfrom skyvern.utils.url_validators import validate_fetch_url+from skyvern.webeye.action_deadline import bounded_lookup_callfrom skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.browser_driver_errors import is_driver_error, is_driver_timeout_errorfrom skyvern.webeye.browser_engine import BrowserEngineSelection@@ -197,23 +199,24 @@ async def resolve_locator(while len(iframe_path) > 0:child_frame = iframe_path.pop()- frame_handler = await current_frame.query_selector(f"[{SKYVERN_ID_ATTR}='{child_frame}']")+ frame_locator_query = f"[{SKYVERN_ID_ATTR}='{child_frame}']"+ frame_handler = await bounded_lookup_call(lambda: current_frame.query_selector(frame_locator_query))if frame_handler is None:raise NoneFrameError(frame_id=child_frame)try:- content_frame = await frame_handler.content_frame()+ content_frame = await bounded_lookup_call(frame_handler.content_frame)except Exception as exc:if not is_engine_error(exc, engine_selection) or not is_element_detached_error(exc):raise# The iframe node detached between query_selector and content_frame (page# mutation). Re-query once for a fresh handle; if it is gone or detaches# again, classify as a missing element instead of leaking the raw error.- frame_handler = await current_frame.query_selector(f"[{SKYVERN_ID_ATTR}='{child_frame}']")+ frame_handler = await bounded_lookup_call(lambda: current_frame.query_selector(frame_locator_query))if frame_handler is None:raise MissingElement(element_id=child_frame) from exctry:- content_frame = await frame_handler.content_frame()+ content_frame = await bounded_lookup_call(frame_handler.content_frame)except Exception as retry_exc:if not is_engine_error(retry_exc, engine_selection) or not is_element_detached_error(retry_exc):raise@@ -265,7 +268,7 @@ class SkyvernElement:frame = incre_page.skyvern_frame.get_frame()locator = frame.locator(css_selector)- num_elements = await locator.count()+ num_elements = await bounded_lookup_call(locator.count)if num_elements < 1:LOG.debug("No elements found with css. Validation failed.", css=css_selector, element_id=element_id)raise MissingElement(selector=css_selector, element_id=element_id)@@ -542,7 +545,7 @@ class SkyvernElement:return await self.get_selectable() or self.get_tag_name() in SELECTABLE_ELEMENTasync def is_visible(self, must_visible_style: bool = True) -> bool:- if not await self.get_locator().count():+ if not await bounded_lookup_call(self.get_locator().count):return Falseif not must_visible_style:return True@@ -676,8 +679,10 @@ class SkyvernElement:parent_locator = hover_target.locator("..")try:- if await parent_locator.count() != 1:+ if await bounded_lookup_call(parent_locator.count) != 1:break+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:LOG.debug("Unable to evaluate parent locator during hover reveal", exc_info=True, element_id=self.get_id()@@ -975,7 +980,7 @@ class SkyvernElement:locator = self.get_frame().locator(f"[id='{for_id}']")# supposed to be only one element, since id is unique in the whole DOM- if await locator.count() != 1:+ if await bounded_lookup_call(locator.count) != 1:return Noneunique_id = await locator.get_attribute(SKYVERN_ID_ATTR, timeout=timeout)@@ -997,7 +1002,9 @@ class SkyvernElement:# fallback: fail_closed=True treats the label as forwarding so it is refused# rather than clicked unvetted.try:- return await label_locator.locator("a[href], button").count() > 0+ return await bounded_lookup_call(label_locator.locator("a[href], button").count) > 0+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:return fail_closed@@ -1016,7 +1023,9 @@ class SkyvernElement:if self.get_tag_name() != "label":return Truetry:- return await locator.locator("a[href], button").count() == 0+ return await bounded_lookup_call(locator.locator("a[href], button").count) == 0+ except SkyvernPageAnalysisTimeout:+ raiseexcept Exception:return False@@ -1029,7 +1038,7 @@ class SkyvernElement:return Nonelocator = self.get_frame().locator(f"label[for='{element_id}']")- cnt = await locator.count()+ cnt = await bounded_lookup_call(locator.count)if cnt == 1:if await self._label_click_forwards_to_descendant(locator):return None@@ -1044,7 +1053,7 @@ class SkyvernElement:return Noneparent_locator = self.get_locator().locator("..")- cnt = await parent_locator.count()+ cnt = await bounded_lookup_call(parent_locator.count)if cnt != 1:return None@@ -1124,7 +1133,7 @@ class SkyvernElement:async def find_file_input_in_children(self) -> Locator | None:"""Sometime the file input is invisible on the page, so it won't exist in the element tree, but it can be found in the DOM."""locator = self.get_locator().locator('input[type="file"]')- if await locator.count() != 1:+ if await bounded_lookup_call(locator.count) != 1:return Nonereturn locator@@ -1181,7 +1190,7 @@ class SkyvernElement:# count() itself raises when the locator's frame detached; that means 0 matches# here, not a new error to surface.try:- return await locator.count()+ return await bounded_lookup_call(locator.count)except Exception as exc:if is_engine_error(exc, self._engine_selection) and is_element_detached_error(exc):return 0@@ -1302,7 +1311,7 @@ class SkyvernElement:)wait_config = await get_or_create_wait_config(task_id, workflow_run_id, organization_id)await asyncio.sleep(get_wait_time(wait_config, "checkbox_retry_delay", default=2.0))- if await self.get_locator().count() == 0:+ if await bounded_lookup_call(self.get_locator().count) == 0:LOG.info("Element is not on the page, the checking should work", element_id=self.get_id())returnawait self.get_locator().check(timeout=timeout)@@ -1326,7 +1335,7 @@ class SkyvernElement:)wait_config = await get_or_create_wait_config(task_id, workflow_run_id, organization_id)await asyncio.sleep(get_wait_time(wait_config, "checkbox_retry_delay", default=2.0))- if await self.get_locator().count() == 0:+ if await bounded_lookup_call(self.get_locator().count) == 0:LOG.info("Element is not on the page, the unchecking should work", element_id=self.get_id())returnawait self.get_locator().uncheck(timeout=timeout)@@ -1750,7 +1759,7 @@ class DomUtil:self.scraped_page, self.page, frame, css, engine_selection=self.engine_selection)- num_elements = await locator.count()+ num_elements = await bounded_lookup_call(locator.count)if num_elements < 1:if not allow_xpath_fallback:raise MissingElement(selector=css, element_id=element_id)@@ -1765,7 +1774,7 @@ class DomUtil:# (maybe) we should validate the element hash to make sure the element is the same?LOG.info("Fallback to locator element by xpath.", xpath=xpath, element_id=element_id, sampling=True)locator = frame_content.locator(f"xpath={xpath}")- num_elements = await locator.count()+ num_elements = await bounded_lookup_call(locator.count)if num_elements < 1:raise MissingElement(selector=xpath, element_id=element_id)# The tag-name xpath can resolve to many repeated nodes (rich text editors,@@ -1796,6 +1805,11 @@ class DomUtil:async def safe_get_skyvern_element_by_id(self, element_id: str) -> SkyvernElement | None:try:return await self.get_skyvern_element_by_id(element_id)+ except SkyvernPageAnalysisTimeout:+ # An unresponsive page is not an absent element: answering None would send the caller on+ # to more driver calls against a renderer that stopped answering. Fail closed instead so+ # the step ends and the next scrape's reload rung gets a chance at the page.+ raiseexcept Exception:LOG.warning("Failed to get skyvern element by id", element_id=element_id, exc_info=True)return None@@ -1846,7 +1860,7 @@ class DomUtil:if not isinstance(css, str) or not css:continuelocator = element.get_frame().locator(css)- if await locator.count() != 1:+ if await bounded_lookup_call(locator.count) != 1:continueowner = SkyvernElement(locator,diff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.pyindex ec953b381..548ef93dd 100644--- a/tests/unit/test_auto_completion_discover_fallback.py+++ b/tests/unit/test_auto_completion_discover_fallback.py@@ -15,14 +15,16 @@ import pytestfrom skyvern.exceptions import NoIncrementalElementFoundForAutoCompletion, NoSuitableAutoCompleteOptionfrom skyvern.forge.sdk.models import StepStatus-from skyvern.webeye.actions.actions import InputOrSelectContext+from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextActionfrom skyvern.webeye.actions.handler import (AutoCompletionResult,choose_auto_completion_dropdown,discover_and_select_from_full_dropdown,+ handle_input_text_action,input_or_auto_complete_input,)from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess+from tests.unit.conftest import make_input_element_mockfrom tests.unit.helpers import make_organization, make_step, make_task# ---------------------------------------------------------------------------@@ -934,3 +936,310 @@ async def test_input_or_auto_complete_skips_discover_for_search_bar() -> None:# Search bar returns None early, never reaching discover fallbackassert result is Nonemock_discover.assert_not_called()+++# ---------------------------------------------------------------------------+# Wall-clock budget on the guessed-value fanout+# ---------------------------------------------------------------------------+++def _budget_attempt_mock(seconds_per_attempt: float, elapsed: dict[str, float]) -> AsyncMock:+ """A probe that advances a fake clock, so tests do not depend on how often it is read."""++ async def _attempt(*_args: object, **_kwargs: object) -> AutoCompletionResult:+ elapsed["seconds"] += seconds_per_attempt+ return AutoCompletionResult(action_result=ActionFailure(exception=Exception("no match")))++ return AsyncMock(side_effect=_attempt)+++async def test_ladder_stops_guessing_once_the_retry_budget_is_spent() -> None:+ """The guessed-value fanout is bounded by wall clock, and discovery still runs.++ Each guess costs a full retype plus a DOM scrape; production attempts reached ~62s apiece, so+ three of them spent over four minutes on one field before discovery was even tried.+ """+ skyvern_el = _mock_skyvern_element()+ elapsed = {"seconds": 0.0}+ attempt = _budget_attempt_mock(40.0, elapsed)++ with (+ patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=attempt),+ patch("skyvern.webeye.actions.handler.app") as mock_app,+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,+ patch("skyvern.webeye.actions.handler.settings.AUTO_COMPLETION_LADDER_BUDGET_SECONDS", 60.0),+ patch(+ "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",+ new=AsyncMock(return_value=ActionSuccess()),+ ) as mock_discover,+ ):+ mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(+ return_value={+ "potential_values": [+ {"value": "New York County"},+ {"value": "Kings County"},+ {"value": "Queens County"},+ ]+ }+ )+ mock_prompt.load_prompt.return_value = "mocked prompt"+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)++ result = await input_or_auto_complete_input(+ input_or_select_context=_make_context(field="County"),+ scraped_page=MagicMock(),+ page=MagicMock(),+ dom=MagicMock(),+ text="New York",+ skyvern_element=skyvern_el,+ step=_STEP,+ task=_TASK,+ is_secret_value=False,+ monotonic=lambda: elapsed["seconds"],+ )++ # Probe (40s) stays under budget so one guess is tried (80s); the remaining two are dropped.+ assert attempt.await_count == 2+ assert isinstance(result, ActionSuccess)+ mock_discover.assert_awaited_once()+++async def test_ladder_skips_the_fanout_when_the_probe_alone_spends_the_budget() -> None:+ """A probe that already overran the budget must not buy LLM guesses it cannot try."""+ skyvern_el = _mock_skyvern_element()+ elapsed = {"seconds": 0.0}+ attempt = _budget_attempt_mock(70.0, elapsed)++ with (+ patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=attempt),+ patch("skyvern.webeye.actions.handler.app") as mock_app,+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,+ patch("skyvern.webeye.actions.handler.settings.AUTO_COMPLETION_LADDER_BUDGET_SECONDS", 60.0),+ patch(+ "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",+ new=AsyncMock(return_value=ActionSuccess()),+ ) as mock_discover,+ ):+ mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": [{"value": "Kings County"}]})+ mock_prompt.load_prompt.return_value = "mocked prompt"+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)++ result = await input_or_auto_complete_input(+ input_or_select_context=_make_context(field="County"),+ scraped_page=MagicMock(),+ page=MagicMock(),+ dom=MagicMock(),+ text="New York",+ skyvern_element=skyvern_el,+ step=_STEP,+ task=_TASK,+ is_secret_value=False,+ monotonic=lambda: elapsed["seconds"],+ )++ assert attempt.await_count == 1+ mock_app.SECONDARY_LLM_API_HANDLER.assert_not_called()+ assert isinstance(result, ActionSuccess)+ mock_discover.assert_awaited_once()+++async def test_ladder_tries_every_guess_when_attempts_are_fast() -> None:+ """A responsive control keeps the full fanout: the budget must not truncate healthy ladders."""+ skyvern_el = _mock_skyvern_element()+ elapsed = {"seconds": 0.0}+ attempt = _budget_attempt_mock(2.0, elapsed)++ with (+ patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=attempt),+ patch("skyvern.webeye.actions.handler.app") as mock_app,+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,+ patch("skyvern.webeye.actions.handler.settings.AUTO_COMPLETION_LADDER_BUDGET_SECONDS", 60.0),+ patch(+ "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",+ new=AsyncMock(return_value=None),+ ),+ ):+ mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(+ return_value={+ "potential_values": [+ {"value": "New York County"},+ {"value": "Kings County"},+ {"value": "Queens County"},+ ]+ }+ )+ mock_prompt.load_prompt.return_value = "mocked prompt"+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)++ result = await input_or_auto_complete_input(+ input_or_select_context=_make_context(field="County"),+ scraped_page=MagicMock(),+ page=MagicMock(),+ dom=MagicMock(),+ text="New York",+ skyvern_element=skyvern_el,+ step=_STEP,+ task=_TASK,+ is_secret_value=False,+ monotonic=lambda: elapsed["seconds"],+ )++ assert attempt.await_count == 4 # the probe plus all three guesses+ assert result is None+++# ---------------------------------------------------------------------------+# An exhausted ladder must not report success over an empty field+# ---------------------------------------------------------------------------+++async def _run_exhausted_ladder_input(+ *,+ read_back: str | None,+ is_search_bar: bool = False,+ incremental_options: list[dict] | None = None,+ committed_selection: bool = False,+ secret_value: str | None = None,+ selection_readback: AsyncMock | None = None,+) -> tuple[list, MagicMock, AsyncMock]:+ """Drive handle_input_text_action with a ladder that selected nothing.++ ``read_back`` is what the field holds once the ladder has given up and the Tab commit has+ fired -- the only signal separating a value that landed from a control that cleared itself+ on blur.+ """+ skyvern_el = make_input_element_mock(element_id="AADC")+ skyvern_el.is_auto_completion_input = AsyncMock(return_value=True)+ dom_instance = MagicMock()+ dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)++ inc = MagicMock()+ inc.start_listen_dom_increment = AsyncMock()+ inc.stop_listen_dom_increment = AsyncMock()+ # Non-empty so the Tab commit hack arms, as it does on a real autocomplete control.+ inc.get_incremental_element_tree = AsyncMock(+ return_value=incremental_options if incremental_options is not None else [{"id": "OPT", "text": "Nassau"}]+ )++ skyvern_frame = MagicMock()+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()++ scraped_page = MagicMock()+ scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}++ context = InputOrSelectContext(field="County", is_search_bar=is_search_bar, is_location_input=False)+ action_text = "{{login.username}}" if secret_value is not None else "New York"+ resolved_text = secret_value if secret_value is not None else "New York"+ action = InputTextAction(element_id="AADC", text=action_text, reasoning="fill the county")+ selection_readback = selection_readback or AsyncMock(+ return_value=(committed_selection, "scope_token" if committed_selection else "none")+ )++ with (+ patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),+ patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),+ patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(return_value=read_back)),+ patch(+ "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",+ return_value=resolved_text,+ ),+ patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),+ patch(+ "skyvern.webeye.actions.handler._is_collapse_autocomplete_fanout_enabled",+ new=AsyncMock(return_value=False),+ ),+ patch("skyvern.webeye.actions.handler._is_combobox_or_typeahead", new=AsyncMock(return_value=False)),+ patch("skyvern.webeye.actions.handler._fill_secret_with_readback", new=AsyncMock(return_value=None)),+ # No in-action custom selection commits, so the ladder is the only thing that could have.+ patch("skyvern.webeye.actions.handler.sequentially_select_from_dropdown", new=AsyncMock(return_value=None)),+ patch(+ "skyvern.webeye.actions.handler._custom_select_scope_confirms_committed",+ new=selection_readback,+ ),+ # The ladder ran every attempt and selected nothing.+ patch("skyvern.webeye.actions.handler.input_or_auto_complete_input", new=AsyncMock(return_value=None)),+ ):+ results = await handle_input_text_action(+ action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP+ )+ return results, skyvern_el, selection_readback+++async def test_exhausted_ladder_over_an_empty_field_fails_the_action() -> None:+ """The ladder selected nothing and the field is empty -> a failed action, not 'Action succeeded'.++ Reporting success here hides a blank required field from the agent, which then keeps filling+ the rest of the form and submits it.+ """+ results, _, _ = await _run_exhausted_ladder_input(read_back="")++ assert len(results) == 1+ assert isinstance(results[0], ActionFailure)+ assert results[0].exception_type == "NoSuitableAutoCompleteOption"+ assert results[0].stop_execution_on_failure is True+ assert results[0].skip_remaining_actions is True+ assert "no matching option was committed" in (results[0].exception_message or "")+ assert "field was left empty" not in (results[0].exception_message or "")+++async def test_exhausted_ladder_that_left_a_value_still_succeeds() -> None:+ """The typed text survived the commit, so the field is filled and the action succeeded."""+ results, _, _ = await _run_exhausted_ladder_input(read_back="New York")++ assert len(results) == 1+ assert isinstance(results[0], ActionSuccess)+++async def test_exhausted_ladder_on_a_search_bar_still_succeeds() -> None:+ """A search bar legitimately holds typed text with nothing selected, so it is not armed."""+ results, _, _ = await _run_exhausted_ladder_input(read_back="", is_search_bar=True)++ assert len(results) == 1+ assert isinstance(results[0], ActionSuccess)+++async def test_exhausted_ladder_blurs_even_when_no_options_were_detected() -> None:+ results, skyvern_el, _ = await _run_exhausted_ladder_input(read_back="", incremental_options=[])++ assert isinstance(results[0], ActionFailure)+ assert ("Tab",) in [call.args for call in skyvern_el.press_key.await_args_list]+++async def test_exhausted_ladder_treats_a_committed_chip_as_filled() -> None:+ results, _, selection_readback = await _run_exhausted_ladder_input(read_back="", committed_selection=True)++ assert len(results) == 1+ assert isinstance(results[0], ActionSuccess)+ assert selection_readback.await_args.kwargs["expected_label"] == "new york"+ assert selection_readback.await_args.kwargs["allow_aria_selected_option_tokens"] is False+++async def test_exhausted_ladder_redacts_secret_from_failure_message() -> None:+ secret_value = "unlogged-sensitive-value-9e2d"+ results, _, selection_readback = await _run_exhausted_ladder_input(read_back="", secret_value=secret_value)++ assert isinstance(results[0], ActionFailure)+ assert secret_value not in (results[0].exception_message or "")+ assert "target_value=<secret>" in (results[0].exception_message or "")+ assert selection_readback.await_args.kwargs["matched_label"] == "<secret>"+++async def test_exhausted_ladder_does_not_treat_an_unreadable_field_as_empty() -> None:+ results, _, _ = await _run_exhausted_ladder_input(read_back=None)++ assert len(results) == 1+ assert isinstance(results[0], ActionSuccess)diff --git a/tests/unit/test_auto_completion_location.py b/tests/unit/test_auto_completion_location.pyindex b3b985afb..76e0a317c 100644--- a/tests/unit/test_auto_completion_location.py+++ b/tests/unit/test_auto_completion_location.py@@ -13,7 +13,11 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom skyvern.constants import SKYVERN_ID_ATTR-from skyvern.exceptions import InvalidElementForTextInput, NoIncrementalElementFoundForAutoCompletion+from skyvern.exceptions import (+ InvalidElementForTextInput,+ NoIncrementalElementFoundForAutoCompletion,+ SkyvernPageAnalysisTimeout,+)from skyvern.forge.sdk.models import StepStatusfrom skyvern.webeye.actions.actions import InputOrSelectContextfrom skyvern.webeye.actions.handler import (@@ -144,6 +148,36 @@ async def test_location_single_option_skips_llm() -> None:assert isinstance(result.action_result, ActionSuccess)+async def test_auto_completion_propagates_page_analysis_timeout() -> None:+ frame = _mock_frame()+ frame.locator.return_value.count = AsyncMock(side_effect=SkyvernPageAnalysisTimeout("lookup timed out"))+ skyvern_el = _mock_skyvern_element(frame)+ inc_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)++ with (+ patch(+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",+ new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),+ ),+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),+ ):+ with pytest.raises(SkyvernPageAnalysisTimeout):+ await choose_auto_completion_dropdown(+ context=_make_location_context(),+ page=MagicMock(),+ scraped_page=MagicMock(),+ dom=MagicMock(),+ text="123 Main St",+ skyvern_element=skyvern_el,+ step=_STEP,+ task=_TASK,+ preserved_elements=[{"id": "preserved"}],+ is_secret_value=False,+ is_location_input=True,+ )++@pytest.mark.asyncioasync def test_location_whitespace_normalized_still_matches() -> None:"""Input with extra whitespace should still match after normalization."""diff --git a/tests/unit/test_click_desired_state.py b/tests/unit/test_click_desired_state.pyindex bc5a92dcd..fca174892 100644--- a/tests/unit/test_click_desired_state.py+++ b/tests/unit/test_click_desired_state.py@@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom playwright.async_api import Page, async_playwright+from skyvern.exceptions import SkyvernPageAnalysisTimeoutfrom skyvern.webeye.actions import handlerfrom skyvern.webeye.actions.actions import ActionStatus, ClickAction, ClickContextfrom skyvern.webeye.actions.responses import ActionAbort, ActionFailure, ActionResult, ActionSuccess, StaleActionAbort@@ -499,6 +500,15 @@ async def test_resolve_read_exception_returns_none() -> None:assert await handler._resolve_live_selected_state(element) is None+async def test_apply_desired_state_propagates_selected_option_lookup_timeout() -> None:+ element = FakeToggleElement("div", {"role": "option", "aria-selected": "true"})+ timeout = SkyvernPageAnalysisTimeout("lookup timed out")+ with patch.object(handler, "bounded_lookup_call", AsyncMock(side_effect=timeout)):+ with pytest.raises(SkyvernPageAnalysisTimeout, match="lookup timed out"):+ await handler._apply_desired_click_state(_click_action(), element, True, MagicMock())++@pytest.mark.parametrize("attributes", [{}, {"role": "button"}, {"href": "https://example.test/next"}])@pytest.mark.asyncioasync def test_resolve_control_without_boolean_observable_returns_none(attributes: dict[str, str]) -> None:@@ -1348,6 +1358,29 @@ def _static_listbox_snapshot(**overrides: object) -> dict[str, object]:return snapshot+async def test_static_listbox_resolution_propagates_lookup_timeout() -> None:+ timeout = SkyvernPageAnalysisTimeout("lookup timed out")+ option = MagicMock()+ option.is_visible = AsyncMock(return_value=True)+ option.element_handle = AsyncMock(return_value=MagicMock())+ option.get_attribute = AsyncMock(return_value="option-id")+ element = MagicMock()+ element.get_locator.return_value.locator.return_value.all = AsyncMock(return_value=[option])+ element.get_frame.return_value = MagicMock()+ element.get_id.return_value = "listbox"+ dom = MagicMock()+ dom.safe_get_skyvern_element_by_id = AsyncMock(side_effect=timeout)++ with patch.object(+ handler, "read_static_listbox_option_snapshot", new=AsyncMock(return_value=_static_listbox_snapshot())+ ):+ with pytest.raises(SkyvernPageAnalysisTimeout, match="lookup timed out"):+ await handler._resolve_static_listbox_container_target(_click_action(), element, dom)++ dom.safe_get_skyvern_element_by_id.assert_awaited_once_with("option-id")++@pytest.mark.parametrize(("snapshot", "expected"),[@@ -1978,6 +2011,17 @@ async def test_apply_grid_row_driver_failure_falls_open() -> None:drive.assert_awaited_once()+async def test_grid_row_driver_propagates_page_analysis_timeout() -> None:+ element = MagicMock()+ cell_locator = MagicMock()+ cell_locator.count = AsyncMock(side_effect=SkyvernPageAnalysisTimeout("lookup timed out"))+ element.get_locator.return_value.locator.return_value.first = cell_locator++ with pytest.raises(SkyvernPageAnalysisTimeout):+ await handler._drive_grid_row_selection(_click_action(), element, True)++@pytest.mark.asyncioasync def test_apply_grid_row_unmarked_desired_true_falls_open_without_driver() -> None:# v3.1 invariant: an UNMARKED row (no positive selected signal) with desired=True is DOM-diff --git a/tests/unit/test_incremental_scrape_lookup_deadline.py b/tests/unit/test_incremental_scrape_lookup_deadline.pynew file mode 100644index 000000000..602555482--- /dev/null+++ b/tests/unit/test_incremental_scrape_lookup_deadline.py@@ -0,0 +1,53 @@+from __future__ import annotations++from collections.abc import Awaitable, Callable+from types import SimpleNamespace++import pytest++from skyvern.exceptions import SkyvernPageAnalysisTimeout+from skyvern.webeye.scraper import scraper as scraper_module+from skyvern.webeye.scraper.scraper import IncrementalScrapePage+++class _CountLocator:+ def __init__(self, selector: str) -> None:+ self.selector = selector++ async def count(self) -> int:+ return 1+++class _Frame:+ def locator(self, selector: str) -> _CountLocator:+ return _CountLocator(selector)++[email protected]("timed_out_element_id", ["child", "parent"])+async def test_select_one_element_by_value_propagates_lookup_deadline(+ monkeypatch: pytest.MonkeyPatch, timed_out_element_id: str+) -> None:+ page = IncrementalScrapePage(SimpleNamespace(get_frame=lambda: _Frame())) # type: ignore[arg-type]+ page.element_tree = [+ {+ "id": "parent",+ "interactable": True,+ "children": [{"id": "child", "text": "target", "interactable": False}],+ }+ ]+ lookups: list[str] = []++ async def bounded_lookup(call: Callable[[], Awaitable[int]]) -> int:+ element_id = "child" if '"child"' in call.__self__.selector else "parent"+ lookups.append(element_id)+ if element_id == timed_out_element_id:+ raise SkyvernPageAnalysisTimeout("lookup timed out")+ return await call()++ monkeypatch.setattr(scraper_module, "bounded_lookup_call", bounded_lookup)++ with pytest.raises(SkyvernPageAnalysisTimeout):+ await page.select_one_element_by_value("target")++ assert lookups[-1] == timed_out_element_iddiff --git a/tests/unit/test_multi_field_totp.py b/tests/unit/test_multi_field_totp.pyindex 490eae1b8..5e23d45e4 100644--- a/tests/unit/test_multi_field_totp.py+++ b/tests/unit/test_multi_field_totp.py@@ -23,6 +23,7 @@ from playwright.async_api import TimeoutError as PlaywrightTimeoutErrorfrom pydantic import TypeAdapterfrom skyvern.config import settings+from skyvern.exceptions import SkyvernPageAnalysisTimeoutfrom skyvern.forge import agent as agent_modulefrom skyvern.forge.agent import (ForgeAgent,@@ -4500,6 +4501,44 @@ async def test_submission_evidence_filters_volatile_nodes_and_rebinds_scopes(assert rebound.evaluate.await_args is not None+async def test_group_binding_propagates_page_analysis_timeout(monkeypatch: pytest.MonkeyPatch) -> None:+ attempt = _attempt()+ group = SimpleNamespace(+ box_element_ids=attempt.box_element_ids,+ frame="main.frame",+ container={"id": "otp-container"},+ )+ fresh = SimpleNamespace(+ url="same",+ _document_loader_id="loader",+ id_to_frame_dict={element_id: "main.frame" for element_id in attempt.box_element_ids},+ )+ original = SimpleNamespace(+ url="same",+ _document_loader_id="loader",+ id_to_frame_dict={element_id: "main.frame" for element_id in attempt.box_element_ids},+ id_to_css_dict={element_id: f"#{element_id}" for element_id in attempt.box_element_ids},+ generate_scraped_page_without_screenshots=AsyncMock(return_value=fresh),+ )+ page = SimpleNamespace(url="same")+ timeout = SkyvernPageAnalysisTimeout("lookup timed out")+ lookup = AsyncMock(side_effect=timeout)+ locator = SimpleNamespace(count=AsyncMock(return_value=1))++ monkeypatch.setattr(multi_field_totp_module, "_document_continuity", AsyncMock(return_value=True))+ monkeypatch.setattr(multi_field_totp_module, "_multi_field_totp_box_groups", lambda *_: [group])+ monkeypatch.setattr(multi_field_totp_module, "_multi_field_totp_container_identity", lambda *_: "same-widget")+ monkeypatch.setattr(multi_field_totp_module, "_multi_field_totp_matching_scopes", lambda *_: [object()])+ monkeypatch.setattr(multi_field_totp_module, "resolve_locator", AsyncMock(return_value=(locator, None)))+ monkeypatch.setattr(multi_field_totp_module, "bounded_lookup_call", lookup)++ with pytest.raises(SkyvernPageAnalysisTimeout, match="lookup timed out"):+ await multi_field_totp_module._refresh_multi_field_totp_group_binding(original, page, attempt)++ lookup.assert_awaited_once()++def test_widget_probe_hashes_own_nodes_without_secret_values_or_descendant_churn() -> None:node = shutil.which("node")if node is None:@@ -4720,6 +4759,16 @@ async def test_fill_observer_preserves_only_counts_across_document_teardown() ->assert state.observed_max_filled == 0+async def test_fill_observer_refresh_propagates_page_analysis_timeout(monkeypatch: pytest.MonkeyPatch) -> None:+ timeout = SkyvernPageAnalysisTimeout("lookup timed out")+ monkeypatch.setattr(multi_field_totp_module, "bounded_lookup_call", AsyncMock(side_effect=timeout))+ observer = multi_field_totp_module.MultiFieldTotpFillObserver(SimpleNamespace(), _attempt(), "test:")++ with pytest.raises(SkyvernPageAnalysisTimeout, match="lookup timed out"):+ await observer.refresh([SimpleNamespace(count=AsyncMock())])++def test_input_observer_measures_peak_fullness_before_site_clear() -> None:for clear_at in (2, 4):observed = _probe_live_fill_observer(remount=False, clear_at=clear_at)diff --git a/tests/unit/test_stale_web_action_remap.py b/tests/unit/test_stale_web_action_remap.pyindex 966cc8a2b..022aaf855 100644--- a/tests/unit/test_stale_web_action_remap.py+++ b/tests/unit/test_stale_web_action_remap.py@@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMockimport pytest+from skyvern.exceptions import SkyvernPageAnalysisTimeoutfrom skyvern.webeye.actions import handlerfrom skyvern.webeye.actions.actions import ClickAction@@ -381,3 +382,25 @@ async def test_indeterminate_probe_falls_through_to_legacy(monkeypatch: pytest.Massert result is Noneassert action.element_id == "OLD"rescrape.assert_not_awaited()+++async def test_stale_remap_propagates_lookup_deadline(monkeypatch: pytest.MonkeyPatch) -> None:+ batch = _scraped({"OLD": _element("Approve", html_id="approve_field")})+ monkeypatch.setattr(handler, "_document_continuity", AsyncMock(return_value=True))+ locator = MagicMock()+ locator.count = AsyncMock(side_effect=SkyvernPageAnalysisTimeout("lookup deadline"))+ monkeypatch.setattr(handler, "resolve_locator", AsyncMock(return_value=(locator, MagicMock())))++ with pytest.raises(SkyvernPageAnalysisTimeout):+ await handler._refresh_stale_web_action_before_dispatch(batch, _live_page(), ClickAction(element_id="OLD"))+++async def test_stale_batch_probe_propagates_lookup_deadline(monkeypatch: pytest.MonkeyPatch) -> None:+ batch = _scraped({"OLD": _element("Approve", html_id="approve_field")})+ monkeypatch.setattr(handler, "_document_continuity", AsyncMock(return_value=True))+ locator = MagicMock()+ locator.count = AsyncMock(side_effect=SkyvernPageAnalysisTimeout("lookup deadline"))+ monkeypatch.setattr(handler, "resolve_locator", AsyncMock(return_value=(locator, MagicMock())))++ with pytest.raises(SkyvernPageAnalysisTimeout):+ await handler._batched_target_stale_beyond_remap(batch, _live_page(), ClickAction(element_id="OLD"))diff --git a/tests/unit/test_xpath_fallback_ambiguity.py b/tests/unit/test_xpath_fallback_ambiguity.pyindex 993d14aef..99f3a5feb 100644--- a/tests/unit/test_xpath_fallback_ambiguity.py+++ b/tests/unit/test_xpath_fallback_ambiguity.py@@ -1,10 +1,13 @@from __future__ import annotations+import asynciofrom unittest.mock import AsyncMock, MagicMockimport pytest-from skyvern.exceptions import MissingElement, MultipleElementsFound+from skyvern.config import settings+from skyvern.exceptions import MissingElement, MultipleElementsFound, SkyvernPageAnalysisTimeout+from skyvern.webeye import action_deadlinefrom skyvern.webeye.utils.dom import DomUtil@@ -18,7 +21,7 @@ def _make_dom() -> DomUtil:return DomUtil(scraped_page, MagicMock())-def _patch_resolve(monkeypatch: pytest.MonkeyPatch, *, css_count: int, xpath_count: int) -> MagicMock:+def _patch_resolve(monkeypatch: pytest.MonkeyPatch, *, css_count: int, xpath_count: int) -> tuple[MagicMock, MagicMock]:css_locator = MagicMock()css_locator.count = AsyncMock(return_value=css_count)xpath_locator = MagicMock()@@ -32,7 +35,7 @@ def _patch_resolve(monkeypatch: pytest.MonkeyPatch, *, css_count: int, xpath_coureturn css_locator, frame_contentmonkeypatch.setattr("skyvern.webeye.utils.dom.resolve_locator", _resolve)- return xpath_locator+ return css_locator, xpath_locator@pytest.mark.asyncio@@ -61,3 +64,42 @@ async def test_css_multiple_matches_still_rejected(monkeypatch: pytest.MonkeyPat_patch_resolve(monkeypatch, css_count=2, xpath_count=1)with pytest.raises(MultipleElementsFound):await _make_dom().get_skyvern_element_by_id("el-1")+++async def test_lookup_against_a_frozen_page_ends_at_its_deadline(monkeypatch: pytest.MonkeyPatch) -> None:+ """``count()`` takes no Playwright timeout, so a renderer that stopped answering held one action+ for 15.5 minutes until the page recovered on its own."""+ monkeypatch.setattr(action_deadline, "ACTION_DEADLINE_HEADROOM_MS", 0)+ monkeypatch.setattr(settings, "BROWSER_ELEMENT_LOOKUP_TIMEOUT_MS", 50)+ css_locator, _ = _patch_resolve(monkeypatch, css_count=0, xpath_count=1)+ frozen = asyncio.Event()++ async def never_answers() -> int:+ await frozen.wait()+ return 0++ css_locator.count = never_answers++ with pytest.raises(SkyvernPageAnalysisTimeout):+ # A lookup that ignores its budget fails the test here instead of hanging the suite.+ async with asyncio.timeout(10):+ await _make_dom().get_skyvern_element_by_id("el-1")+++async def test_safe_lookup_propagates_analysis_timeout(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(action_deadline, "ACTION_DEADLINE_HEADROOM_MS", 0)+ monkeypatch.setattr(settings, "BROWSER_ELEMENT_LOOKUP_TIMEOUT_MS", 50)+ css_locator, _ = _patch_resolve(monkeypatch, css_count=0, xpath_count=1)+ frozen = asyncio.Event()++ async def never_answers() -> int:+ await frozen.wait()+ return 0++ css_locator.count = never_answers++ with pytest.raises(SkyvernPageAnalysisTimeout):+ async with asyncio.timeout(10):+ await _make_dom().safe_get_skyvern_element_by_id("el-1")