Skip to content

Automate browser based workflows with AI

Bound the engine 1.0 element lookup so a frozen page cannot stall an action (#8990)

Aaron Perez73e49ce7bb3d
diff --git a/skyvern/config.py b/skyvern/config.py
index b57d85286..940d39abb 100644
--- a/skyvern/config.py
+++ b/skyvern/config.py
@@ -141,6 +141,11 @@ class Settings(BaseSettings):
TEMP_PATH: str = "./temp"
DOWNLOAD_PATH: str = f"{REPO_ROOT_DIR}/downloads"
BROWSER_ACTION_TIMEOUT_MS: int = 5000
+ # Playwright's `Locator.count()` and the iframe-chain `query_selector`/`content_frame` probes take
+ # no timeout argument, so a renderer that stopped answering leaves each one waiting on a protocol
+ # reply forever. The action deadline adds its headroom to this budget; expiry means the page stopped
+ # answering, not that a healthy lookup was merely slow.
+ BROWSER_ELEMENT_LOOKUP_TIMEOUT_MS: int = 5000
BROWSER_ACTION_MAX_EXECUTION_SECONDS: int = 1200
WORKER_STALL_DUMP_SECONDS: int = 0
# "enforce" is deliberately absent: the policy core can only observe until every sink consumes
@@ -181,6 +186,12 @@ class Settings(BaseSettings):
# picks from whatever loaded — so this is kept short enough that a never-settling menu costs
# seconds rather than the minutes a run's elapsed budget cannot absorb.
OPTION_LOADING_TIMEOUT_MS: int = 60000
+ # Wall-clock budget for the autocomplete guessed-value fanout, measured from the ladder's first
+ # probe. The probe for the requested value always runs; this only gates the LLM-guessed retries,
+ # each of which costs a full clear-and-retype plus a DOM scrape. Without it a control whose
+ # options never match spent minutes per field before the discovery fallback was reached. The budget
+ # stops new guesses from starting; a guess already in progress may overrun it. 0 disables this limit.
+ AUTO_COMPLETION_LADDER_BUDGET_SECONDS: float = 60.0
MAX_STEPS_PER_RUN: int = 10
MAX_STEPS_PER_TASK_V2: int = 25
MAX_ITERATIONS_PER_TASK_V2: int = 50
diff --git a/skyvern/webeye/action_deadline.py b/skyvern/webeye/action_deadline.py
index e29d0083c..4b5db0c6e 100644
--- a/skyvern/webeye/action_deadline.py
+++ b/skyvern/webeye/action_deadline.py
@@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable
from contextlib import asynccontextmanager
from typing import Any, TypeVar
+from skyvern.config import settings
from skyvern.exceptions import ActionDeadlineExceeded
from skyvern.forge.sdk.core import skyvern_context
from skyvern.webeye.browser_health import BrowserOperation
@@ -18,6 +19,15 @@ T = TypeVar("T")
_UNSET = object()
+async def bounded_lookup_call(call: Callable[[], Awaitable[T]]) -> T:
+ """Bound a lookup call without a native Playwright timeout, such as ``Locator.count()``."""
+ async with under_action_deadline(
+ budget_ms=settings.BROWSER_ELEMENT_LOOKUP_TIMEOUT_MS,
+ operation=BrowserOperation.EVALUATE,
+ ):
+ return await cancellable_driver_call(call)
+
+
def cancellation_pending() -> bool:
"""Whether this task carries an unhandled cancellation request.
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index faa8b4c83..34493cf9b 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -274,6 +274,7 @@ from skyvern.webeye.utils.dom import (
InteractiveElement,
SkyvernElement,
SkyvernOptionType,
+ bounded_lookup_call,
is_element_detached_error,
is_incompatible_text_input_error,
is_post_dispatch_click_timeout,
@@ -2655,7 +2656,7 @@ class CustomSingleSelectResult:
if not isinstance(self.action_result, ActionSuccess):
return True
- if await self.dropdown_menu.get_locator().count() == 0:
+ if await bounded_lookup_call(self.dropdown_menu.get_locator().count) == 0:
return True
return not await self.skyvern_frame.get_element_visible(self.dropdown_menu.get_locator())
@@ -6435,8 +6436,10 @@ async def _batched_target_stale_beyond_remap(
return False # only the main frame has a scrape-stable identity to reason about
try:
locator, frame_content = await resolve_locator(scraped_page, page, frame, css)
- if await locator.count() == 1:
+ if await bounded_lookup_call(locator.count) == 1:
return False # the exact injected node is still live -> not stale
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
return False # cannot confirm staleness -> decline; legacy dispatch stays authoritative
return True
@@ -6497,8 +6500,10 @@ async def _refresh_stale_web_action_before_dispatch(
return None
try:
locator, frame_content = await resolve_locator(scraped_page, page, frame, css)
- if await locator.count() == 1:
+ if await bounded_lookup_call(locator.count) == 1:
return None # the exact injected node is still live -> not stale -> dispatch unchanged
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
return None # cannot confirm liveness / marker survival -> decline; legacy path authoritative
@@ -6643,7 +6648,7 @@ async def _is_single_select_option_highlight(element: SkyvernElement) -> bool:
if (await element.get_attr("role", mode="dynamic") or "").strip().casefold() != "option":
return False
multiselectable = element.get_locator().locator("xpath=ancestor-or-self::*[@aria-multiselectable][1]")
- if await multiselectable.count() == 0:
+ if await bounded_lookup_call(multiselectable.count) == 0:
return True
return (await multiselectable.get_attribute("aria-multiselectable") or "").strip().casefold() != "true"
@@ -6698,6 +6703,8 @@ async def _resolve_static_listbox_container_target(
if target is None or not await _is_static_listbox_option(target) or await target.is_disabled(dynamic=True):
return _static_listbox_replan_failure(action, "The static listbox option is unavailable; rescrape it.")
return target
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
LOG.debug("Static listbox container inspection failed", element_id=element.get_id())
return None
@@ -6991,7 +6998,7 @@ async def _drive_grid_row_selection(
click does not achieve the desired selection. Never loops, force-clicks, or mutates the DOM."""
cell_locator = element.get_locator().locator('xpath=ancestor-or-self::*[@role="gridcell" or self::td][1]').first
try:
- if await cell_locator.count() == 0:
+ if await bounded_lookup_call(cell_locator.count) == 0:
return None
if await SkyvernElement._label_click_forwards_to_descendant(cell_locator, fail_closed=True):
LOG.warning(
@@ -7007,6 +7014,8 @@ async def _drive_grid_row_selection(
)
return None
await cell_locator.click(timeout=settings.BROWSER_ACTION_TIMEOUT_MS, position=point)
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
LOG.warning(
"Grid row-selection click failed, continuing the normal click", element_id=element.get_id(), exc_info=True
@@ -7080,6 +7089,8 @@ async def _resolve_live_click_state(
LOG.info("Trusting aria-selected on a static listbox option", element_id=element.get_id())
return _EffectiveClickState(checked=parsed, control_type=None)
return _EffectiveClickState(checked=None, control_type=None)
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
LOG.debug("Failed to read live selected state; continuing with an ordinary click", element_id=element.get_id())
return _EffectiveClickState(checked=None, control_type=None)
@@ -7091,7 +7102,7 @@ async def _resolve_live_selected_state(element: SkyvernElement) -> bool | None:
async def _get_associated_checkbox_label_locator(element: SkyvernElement) -> Locator | None:
ancestor_label_locator = element.get_locator().locator("xpath=ancestor::label[1]")
- if await ancestor_label_locator.count() > 0:
+ if await bounded_lookup_call(ancestor_label_locator.count) > 0:
return ancestor_label_locator
input_id = await element.get_attr("id", mode="dynamic")
@@ -7099,7 +7110,7 @@ async def _get_associated_checkbox_label_locator(element: SkyvernElement) -> Loc
return None
explicit_label_locator = element.get_frame().locator(f"label[for={json.dumps(input_id)}]")
- if await explicit_label_locator.count() > 0:
+ if await bounded_lookup_call(explicit_label_locator.count) > 0:
return explicit_label_locator.first
return None
@@ -9080,6 +9091,59 @@ def _emit_tel_input_outcome(
action.tel_input_outcome = None
+async def _press_autocomplete_commit_tab(*, skyvern_element: SkyvernElement, action: actions.Action) -> None:
+ if not await skyvern_element.is_visible() or await skyvern_element.is_raw_input() or action.skip_auto_complete_tab:
+ return
+ LOG.debug(
+ "Trigger input-selection hack, pressing Tab to choose one",
+ action=action,
+ )
+ await skyvern_element.press_key("Tab")
+
+
+async def _autocomplete_left_field_empty(
+ *,
+ skyvern_element: SkyvernElement,
+ tag_name: str,
+ engine_selection: BrowserEngineSelection | None,
+ expected_value: str,
+ is_secret_value: bool,
+) -> bool:
+ try:
+ value = await get_input_value(
+ tag_name=tag_name,
+ locator=skyvern_element.get_locator(),
+ engine_selection=engine_selection,
+ # This runs on the already-slow exhausted-ladder path, so it must not inherit
+ # Playwright's 30s default wait on a control the commit may have detached.
+ read_timeout_ms=settings.BROWSER_ACTION_TIMEOUT_MS,
+ )
+ except Exception:
+ # A node that cannot be read -- detached, renavigated, or type-incompatible -- is no evidence
+ # of an empty field, and treating it as one would fail actions whose value did land.
+ LOG.info(
+ "Could not read the auto-completion field back, not treating it as empty",
+ element_id=skyvern_element.get_id(),
+ exc_info=True,
+ )
+ return False
+ if value is None:
+ return False
+ if value.strip():
+ return False
+
+ committed, _ = await _custom_select_scope_confirms_committed(
+ readback_scope_element=skyvern_element,
+ anchor_is_combobox_input=tag_name == InteractiveElement.INPUT,
+ matched_element_id=skyvern_element.get_id(),
+ matched_label="<secret>" if is_secret_value else expected_value,
+ expected_label=_normalize_select_shadow_text(expected_value),
+ allow_aria_selected_option_tokens=False,
+ allow_single_value_scope=tag_name == InteractiveElement.INPUT,
+ )
+ return not committed
+
+
@traced(name="skyvern.agent.action.input_text")
async def handle_input_text_action(
action: actions.InputTextAction,
@@ -9367,6 +9431,9 @@ async def _handle_input_text_action(
incremental_element: list[dict] = []
auto_complete_hacky_flag: bool = False
+ # Set when the auto-completion ladder ran every attempt and committed no option, so the plain
+ # fill below is typing text that matched nothing in the control's list.
+ autocomplete_ladder_unresolved: bool = False
# Set when the typeahead prefilter (below) types the target into the field. If that pre-input
# selection does not commit, the terminal fill must clear first so the typed-but-uncommitted value
# is not doubled. `prefilter_attempted` stays set even when the typing raises mid-dispatch (leaving a
@@ -9597,9 +9664,9 @@ async def _handle_input_text_action(
"Find a blocking element to the current element, going to blur the blocking element first",
blocking_element=blocking_element.get_locator(),
)
- if await blocking_element.get_locator().count():
+ if await bounded_lookup_call(blocking_element.get_locator().count):
await blocking_element.press_key("Escape")
- if await blocking_element.get_locator().count():
+ if await bounded_lookup_call(blocking_element.get_locator().count):
await blocking_element.blur()
if try_to_quit_dropdown and await skyvern_element.is_visible():
@@ -9977,6 +10044,9 @@ async def _handle_input_text_action(
):
auto_complete_hacky_flag = False
return [result]
+ # A search bar legitimately keeps free text with nothing selected, so only a
+ # non-search-bar control can have been left blank by the exhausted ladder.
+ autocomplete_ladder_unresolved = not input_or_select_context.is_search_bar
# Only the bare-digit NANP path uses the verified fallback ladder; other tel shapes remain
# observational to preserve flag-off behavior.
@@ -10201,7 +10271,7 @@ async def _handle_input_text_action(
if restored
else "its previous text could not be put back"
),
- target_value=text,
+ target_value="<secret>" if is_secret_value else text,
)
)
]
@@ -10349,6 +10419,34 @@ async def _handle_input_text_action(
# Always stop listening
await incremental_scraped.stop_listen_dom_increment()
+ if autocomplete_ladder_unresolved:
+ # Commit here rather than in the outer finally: the Tab is what blurs the control, and an
+ # autocomplete that rejects unmatched text clears itself on blur. Only a read-back taken
+ # after the commit can confirm that a matching value was committed.
+ if auto_complete_hacky_flag:
+ auto_complete_hacky_flag = False
+ await _press_autocomplete_commit_tab(skyvern_element=skyvern_element, action=action)
+ if await _autocomplete_left_field_empty(
+ skyvern_element=skyvern_element,
+ tag_name=tag_name,
+ engine_selection=engine_selection,
+ expected_value=text,
+ is_secret_value=is_secret_value,
+ ):
+ LOG.info(
+ "Auto completion committed no matching option, reporting a failed action",
+ element_id=skyvern_element.get_id(),
+ )
+ failure = ActionFailure(
+ NoSuitableAutoCompleteOption(
+ reasoning="the auto-completion options matched no attempted value and no matching option was committed",
+ target_value="<secret>" if is_secret_value else text,
+ )
+ )
+ # Prevent a duplicate-element action such as Submit from following this failure.
+ failure.skip_remaining_actions = True
+ return [failure]
+
return [ActionSuccess()]
except (SkyvernPageAnalysisTimeout, InvalidElementForTextInput):
raise
@@ -10375,17 +10473,8 @@ async def _handle_input_text_action(
tel_outcome.browser_valid = await _probe_tel_browser_validity(skyvern_element.get_locator())
# HACK: force to finish missing auto completion input
- if (
- auto_complete_hacky_flag
- and await skyvern_element.is_visible()
- and not await skyvern_element.is_raw_input()
- and not action.skip_auto_complete_tab
- ):
- LOG.debug(
- "Trigger input-selection hack, pressing Tab to choose one",
- action=action,
- )
- await skyvern_element.press_key("Tab")
+ if auto_complete_hacky_flag:
+ await _press_autocomplete_commit_tab(skyvern_element=skyvern_element, action=action)
_URL_RECOVERY_EDIT_DISTANCE_FRACTION = 0.1
@@ -12565,7 +12654,7 @@ async def choose_auto_completion_dropdown(
if not element_id:
continue
locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')
- cnt = await locator.count()
+ cnt = await bounded_lookup_call(locator.count)
if cnt == 0:
continue
@@ -12605,7 +12694,7 @@ async def choose_auto_completion_dropdown(
matched_element_id = matched_candidate.get("element_id") or ""
matched_label = matched_candidate.get("label") or ""
matched_locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{matched_element_id}"]')
- if matched_element_id and matched_label and await matched_locator.count() > 0:
+ if matched_element_id and matched_label and await bounded_lookup_call(matched_locator.count) > 0:
option_identity_matches = await _verify_autocomplete_option_identity(
skyvern_frame=skyvern_frame,
locator=matched_locator,
@@ -12734,7 +12823,7 @@ async def choose_auto_completion_dropdown(
input_normalized = " ".join(text.lower().split())
if fast_path_element_id and input_normalized and input_normalized in option_text:
fast_path_locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{fast_path_element_id}"]')
- if await fast_path_locator.count() > 0:
+ if await bounded_lookup_call(fast_path_locator.count) > 0:
LOG.info(
"Location auto-completion fast path: single option found, skipping LLM",
element_id=fast_path_element_id,
@@ -12824,7 +12913,10 @@ async def choose_auto_completion_dropdown(
if not element_id:
reasoning = json_response.get("reasoning")
- raise NoSuitableAutoCompleteOption(reasoning=reasoning, target_value=text)
+ raise NoSuitableAutoCompleteOption(
+ reasoning=reasoning,
+ target_value="<secret>" if is_secret_value else text,
+ )
if relevance_float < relevance_threshold:
LOG.info(
@@ -12846,7 +12938,7 @@ async def choose_auto_completion_dropdown(
)
locator = current_frame.locator(f'[{SKYVERN_ID_ATTR}="{element_id}"]')
- if await locator.count() == 0:
+ if await bounded_lookup_call(locator.count) == 0:
raise MissingElement(element_id=element_id)
# Use SkyvernElement.click() so we get the full fallback chain
@@ -12876,6 +12968,8 @@ async def choose_auto_completion_dropdown(
clear_input = False
return result
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception as e:
LOG.info(
"Failed to choose the auto completion dropdown",
@@ -12927,6 +13021,7 @@ async def input_or_auto_complete_input(
collapse_autocomplete_fanout_enabled: bool = False,
*,
is_secret_value: bool,
+ monotonic: Callable[[], float] = time.monotonic,
) -> ActionResult | None:
action_phase.mark_action_phase(ActionPhase.AUTOCOMPLETE)
LOG.info(
@@ -12945,6 +13040,26 @@ async def input_or_auto_complete_input(
current_value = text
result = AutoCompletionResult()
+ ladder_started_at = monotonic()
+ retry_budget_seconds = settings.AUTO_COMPLETION_LADDER_BUDGET_SECONDS
+ # Per-attempt value and duration. An attempt is silent for its whole runtime -- no log, no span --
+ # so without this a ladder that spends minutes is indistinguishable from one that spends seconds.
+ attempt_timings: list[dict[str, str | float]] = []
+
+ def _elapsed_seconds() -> float:
+ return monotonic() - ladder_started_at
+
+ def _retry_budget_exhausted() -> bool:
+ return retry_budget_seconds > 0 and _elapsed_seconds() >= retry_budget_seconds
+
+ def _record_attempt(value: str, started_at: float) -> None:
+ attempt_timings.append(
+ {
+ "value": "<secret>" if is_secret_value else value,
+ "seconds": round(monotonic() - started_at, 1),
+ }
+ )
+
while current_attemp < MAX_AUTO_COMPLETE_ATTEMP:
current_attemp += 1
whole_new_elements: list[dict] = []
@@ -12956,6 +13071,7 @@ async def input_or_auto_complete_input(
input_value=current_value,
)
is_location = input_or_select_context.is_location_input or False
+ attempt_started_at = monotonic()
result = await choose_auto_completion_dropdown(
context=input_or_select_context,
page=page,
@@ -12971,6 +13087,7 @@ async def input_or_auto_complete_input(
action=action,
is_secret_value=is_secret_value,
)
+ _record_attempt(current_value, attempt_started_at)
if isinstance(result.action_result, ActionSuccess):
return result.action_result
@@ -12995,29 +13112,40 @@ async def input_or_auto_complete_input(
)
values: list[dict] = []
- prompt = prompt_engine.load_prompt(
- "auto-completion-potential-answers",
- potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,
- field_information=field_information,
- current_value=current_value,
- navigation_goal=task.navigation_goal,
- navigation_payload_str=json.dumps(task.navigation_payload),
- local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
- )
+ if _retry_budget_exhausted():
+ # Guesses the ladder cannot afford to type are not worth asking an LLM for; the
+ # discovery fallback below opens the unfiltered control instead.
+ LOG.info(
+ "Auto completion retry budget already spent by the first probe, skipping the potential-value fanout",
+ element_id=skyvern_element.get_id(),
+ elapsed_seconds=round(_elapsed_seconds(), 1),
+ budget_seconds=retry_budget_seconds,
+ attempts=attempt_timings,
+ )
+ else:
+ prompt = prompt_engine.load_prompt(
+ "auto-completion-potential-answers",
+ potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,
+ field_information=field_information,
+ current_value=current_value,
+ navigation_goal=task.navigation_goal,
+ navigation_payload_str=json.dumps(task.navigation_payload),
+ local_datetime=datetime.now(skyvern_context.ensure_context().tz_info).isoformat(),
+ )
- LOG.info(
- "Ask LLM to give potential values based on the current value",
- current_value=current_value,
- potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,
- )
- if collapse_autocomplete_fanout_enabled and action is not None:
- action.set_has_mini_agent()
- json_respone = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
- prompt=prompt, step=step, prompt_name="auto-completion-potential-answers"
- )
- values = json_respone.get("potential_values", [])
+ LOG.info(
+ "Ask LLM to give potential values based on the current value",
+ current_value=current_value,
+ potential_value_count=AUTO_COMPLETION_POTENTIAL_VALUES_COUNT,
+ )
+ if collapse_autocomplete_fanout_enabled and action is not None:
+ action.set_has_mini_agent()
+ json_respone = await get_org_aware_secondary_llm_api_handler(default=app.SECONDARY_LLM_API_HANDLER)(
+ prompt=prompt, step=step, prompt_name="auto-completion-potential-answers"
+ )
+ values = json_respone.get("potential_values", [])
- for each_value in values:
+ for value_index, each_value in enumerate(values):
value: str = each_value.get("value", "")
if not value:
LOG.info(
@@ -13025,11 +13153,22 @@ async def input_or_auto_complete_input(
value=each_value,
)
continue
+ if _retry_budget_exhausted():
+ LOG.info(
+ "Auto completion retry budget exhausted, skipping the remaining potential values",
+ element_id=skyvern_element.get_id(),
+ elapsed_seconds=round(_elapsed_seconds(), 1),
+ budget_seconds=retry_budget_seconds,
+ attempts=attempt_timings,
+ skipped_value_count=len(values) - value_index,
+ )
+ break
LOG.info(
"Try the potential value for auto completion",
sampling=True,
input_value=value,
)
+ attempt_started_at = monotonic()
result = await choose_auto_completion_dropdown(
context=input_or_select_context,
page=page,
@@ -13046,6 +13185,7 @@ async def input_or_auto_complete_input(
allow_full_page_rescrape=not initial_probe_found_no_options,
is_secret_value=is_secret_value,
)
+ _record_attempt(value, attempt_started_at)
if isinstance(result.action_result, ActionSuccess):
return result.action_result
@@ -13109,6 +13249,8 @@ async def input_or_auto_complete_input(
"Auto completion didn't finish, this might leave the input value to be empty.",
sampling=True,
context=input_or_select_context,
+ attempts=attempt_timings,
+ total_seconds=round(_elapsed_seconds(), 1),
)
return None
@@ -14006,9 +14148,11 @@ async def _evaluate_element_scoped(
async def _read_custom_select_matched_state(element: SkyvernElement) -> dict | None:
try:
- if await element.get_locator().count() != 1:
+ if await bounded_lookup_call(element.get_locator().count) != 1:
return None
state = await _evaluate_element_scoped(element, _CUSTOM_SELECT_MATCHED_STATE_JS)
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
LOG.info(
"Failed to read custom-select matched element state",
diff --git a/skyvern/webeye/actions/multi_field_totp.py b/skyvern/webeye/actions/multi_field_totp.py
index 5b2d83344..cd082302c 100644
--- a/skyvern/webeye/actions/multi_field_totp.py
+++ b/skyvern/webeye/actions/multi_field_totp.py
@@ -15,13 +15,14 @@ import structlog
from playwright.async_api import Locator, Page
from skyvern.constants import SKYVERN_ID_ATTR
+from skyvern.exceptions import SkyvernPageAnalysisTimeout
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.core.skyvern_context import MultiFieldTotpAttempt
from skyvern.utils.contained_effects import contained_effect
from skyvern.webeye.scraper.scraped_page import ScrapedPage
from skyvern.webeye.scraper.scraper import structural_identity
from skyvern.webeye.utils.document import get_main_document_loader_id
-from skyvern.webeye.utils.dom import resolve_locator
+from skyvern.webeye.utils.dom import bounded_lookup_call, resolve_locator
from skyvern.webeye.utils.page import SkyvernFrame
LOG = structlog.get_logger()
@@ -499,7 +500,7 @@ async def _refresh_multi_field_totp_group_binding(
if not css or not frame:
return classified("box_locator_metadata_missing", MultiFieldTotpBindingFailure.UNCONFIRMED)
locator, _ = await resolve_locator(scraped_page, page, frame, css)
- count = await locator.count()
+ count = await bounded_lookup_call(locator.count)
if count not in (0, 1):
return classified("original_box_count_ambiguous", MultiFieldTotpBindingFailure.UNCONFIRMED)
live_boxes.append(count == 1)
@@ -523,7 +524,7 @@ async def _refresh_multi_field_totp_group_binding(
if not container_css or not container_frame or container_frame != original_group.frame:
return classified("container_locator_metadata_missing", MultiFieldTotpBindingFailure.UNCONFIRMED)
locator, _ = await resolve_locator(scraped_page, page, container_frame, container_css)
- container_count = await locator.count()
+ container_count = await bounded_lookup_call(locator.count)
if container_count not in (0, 1):
return classified("original_container_count_ambiguous", MultiFieldTotpBindingFailure.UNCONFIRMED)
container_live = container_count == 1
@@ -585,6 +586,8 @@ async def _refresh_multi_field_totp_group_binding(
if not loader_id_known:
return classified("loader_id_unknown", MultiFieldTotpBindingFailure.UNCONFIRMED)
return fresh, group.box_element_ids
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
return classified("observation_failed", MultiFieldTotpBindingFailure.UNCONFIRMED)
@@ -712,7 +715,7 @@ class MultiFieldTotpFillObserver:
async def refresh(self, boxes: list[Locator]) -> None:
try:
- if not boxes or await boxes[0].count() != 1:
+ if not boxes or await bounded_lookup_call(boxes[0].count) != 1:
return
count = await boxes[0].evaluate(
"(box, prefix) => box.ownerDocument.__skyvernOtpFillObserver?.prefix === prefix ? Number(box.ownerDocument.documentElement.getAttribute('data-skyvern-otp-max-filled')) : 0",
@@ -721,6 +724,8 @@ class MultiFieldTotpFillObserver:
)
if isinstance(count, int) and not isinstance(count, bool) and 0 <= count <= self.attempt.expected_digits:
self.attempt.observed_max_filled = max(self.attempt.observed_max_filled, count)
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
# The console count survives destruction of the document that held the attribute.
pass
@@ -782,13 +787,15 @@ async def _multi_field_totp_widget_scopes(boxes: list[Locator], *, include_ances
if not boxes:
return []
form = boxes[0].locator("xpath=ancestor::form[1]")
- if await form.count() == 1 and all([await form.filter(has=box).count() == 1 for box in boxes]):
+ if await bounded_lookup_call(form.count) == 1 and all(
+ [await bounded_lookup_call(form.filter(has=box).count) == 1 for box in boxes]
+ ):
return [form]
scopes = []
ancestors = boxes[0].locator("xpath=ancestor::*[not(self::body or self::html)]")
- for index in reversed(range(await ancestors.count())):
+ for index in reversed(range(await bounded_lookup_call(ancestors.count))):
ancestor = ancestors.nth(index)
- if all([await ancestor.filter(has=box).count() == 1 for box in boxes]):
+ if all([await bounded_lookup_call(ancestor.filter(has=box).count) == 1 for box in boxes]):
scopes.append(ancestor)
if not include_ancestors or len(scopes) == 3:
break
@@ -992,7 +999,7 @@ async def multi_field_totp_submission_evidence(
boxes: list[Locator] = []
try:
boxes = await _retry_box_locators(page, scraped_page, attempt.box_element_ids)
- counts = [await box.count() for box in boxes]
+ counts = [await bounded_lookup_call(box.count) for box in boxes]
if not all(count == 1 for count in counts):
binding = await _refresh_multi_field_totp_group_binding(scraped_page, page, attempt)
if isinstance(binding, MultiFieldTotpBindingFailure):
@@ -1034,7 +1041,9 @@ async def _multi_field_totp_document_changed(page: Page, original_url: str, old_
async def _multi_field_totp_boxes_disappeared(boxes: list[Locator]) -> bool:
try:
- return bool(boxes) and all([await box.count() == 0 for box in boxes])
+ return bool(boxes) and all([await bounded_lookup_call(box.count) == 0 for box in boxes])
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
return False
@@ -1155,7 +1164,7 @@ async def submit_multi_field_totp_retry(
candidates_considered += 1
control, fields = candidate.locator, candidate.log_fields
try:
- if await control.count() != 1:
+ if await bounded_lookup_call(control.count) != 1:
record("trial", "failed", fields, reason="submit_control_not_actionable")
continue
await dispatch(control, "trial", fields)
@@ -1164,9 +1173,11 @@ async def submit_multi_field_totp_retry(
with timed("pre_evidence_ms"):
if await multi_field_totp_submission_evidence(page, scraped_page, attempt):
return True
- if await control.count() != 1:
+ if await bounded_lookup_call(control.count) != 1:
record("click", "failed", fields, reason="submit_control_not_actionable")
continue
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception as exc:
last_trial_error = exc
continue
@@ -1174,7 +1185,7 @@ async def submit_multi_field_totp_retry(
break
phase = "click" if chosen is not None else "enter"
if chosen is None:
- if not boxes or await boxes[-1].count() != 1:
+ if not boxes or await bounded_lookup_call(boxes[-1].count) != 1:
record("enter", "failed", {}, reason="submit_control_not_actionable")
raise MultiFieldTotpSubmitControlNotActionable() from last_trial_error
chosen = boxes[-1]
@@ -1184,7 +1195,7 @@ async def submit_multi_field_totp_retry(
with timed("pre_evidence_ms"):
if await multi_field_totp_submission_evidence(page, scraped_page, attempt):
return True
- if await chosen.count() != 1:
+ if await bounded_lookup_call(chosen.count) != 1:
record("enter", "failed", fields, reason="submit_control_not_actionable")
raise MultiFieldTotpSubmitControlNotActionable() from last_trial_error
try:
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index c60fd9de5..9503e3628 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -36,6 +36,7 @@ from skyvern.forge.sdk.trace import apply_context_attrs, traced, traced_span
from skyvern.utils.image_resizer import Resolution
from skyvern.utils.token_counter import approx_count_tokens
from skyvern.utils.url_validators import strip_query_params
+from skyvern.webeye.action_deadline import bounded_lookup_call
from skyvern.webeye.browser_state import BLANK_PAGE_URLS, BrowserState
from skyvern.webeye.scraper.scraped_page import (
CleanupElementTreeFunc,
@@ -1154,7 +1155,7 @@ class IncrementalScrapePage(ElementTreeBuilder):
if is_match:
if child_locator:
return child_locator, True
- if interactable and parent_locator and await parent_locator.count() > 0:
+ if interactable and parent_locator and await bounded_lookup_call(parent_locator.count) > 0:
return parent_locator, True
return None, True
@@ -1165,7 +1166,7 @@ class IncrementalScrapePage(ElementTreeBuilder):
if text != value:
return None, False
- if await parent_locator.count() == 0:
+ if await bounded_lookup_call(parent_locator.count) == 0:
return None, False
if not interactable:
diff --git a/skyvern/webeye/utils/dom.py b/skyvern/webeye/utils/dom.py
index b0f080409..a69102af2 100644
--- a/skyvern/webeye/utils/dom.py
+++ b/skyvern/webeye/utils/dom.py
@@ -33,10 +33,12 @@ from skyvern.exceptions import (
NoElementBoudingBox,
NoneFrameError,
SkyvernException,
+ SkyvernPageAnalysisTimeout,
)
from skyvern.experimentation.wait_utils import get_or_create_wait_config, get_wait_time, scroll_into_view_wait
from skyvern.forge.sdk.event.factory import EventStrategyFactory
from skyvern.utils.url_validators import validate_fetch_url
+from skyvern.webeye.action_deadline import bounded_lookup_call
from skyvern.webeye.actions import handler_utils
from skyvern.webeye.browser_driver_errors import is_driver_error, is_driver_timeout_error
from skyvern.webeye.browser_engine import BrowserEngineSelection
@@ -197,23 +199,24 @@ async def resolve_locator(
while len(iframe_path) > 0:
child_frame = iframe_path.pop()
- frame_handler = await current_frame.query_selector(f"[{SKYVERN_ID_ATTR}='{child_frame}']")
+ frame_locator_query = f"[{SKYVERN_ID_ATTR}='{child_frame}']"
+ frame_handler = await bounded_lookup_call(lambda: current_frame.query_selector(frame_locator_query))
if frame_handler is None:
raise NoneFrameError(frame_id=child_frame)
try:
- content_frame = await frame_handler.content_frame()
+ content_frame = await bounded_lookup_call(frame_handler.content_frame)
except Exception as exc:
if not is_engine_error(exc, engine_selection) or not is_element_detached_error(exc):
raise
# The iframe node detached between query_selector and content_frame (page
# mutation). Re-query once for a fresh handle; if it is gone or detaches
# again, classify as a missing element instead of leaking the raw error.
- frame_handler = await current_frame.query_selector(f"[{SKYVERN_ID_ATTR}='{child_frame}']")
+ frame_handler = await bounded_lookup_call(lambda: current_frame.query_selector(frame_locator_query))
if frame_handler is None:
raise MissingElement(element_id=child_frame) from exc
try:
- content_frame = await frame_handler.content_frame()
+ content_frame = await bounded_lookup_call(frame_handler.content_frame)
except Exception as retry_exc:
if not is_engine_error(retry_exc, engine_selection) or not is_element_detached_error(retry_exc):
raise
@@ -265,7 +268,7 @@ class SkyvernElement:
frame = incre_page.skyvern_frame.get_frame()
locator = frame.locator(css_selector)
- num_elements = await locator.count()
+ num_elements = await bounded_lookup_call(locator.count)
if num_elements < 1:
LOG.debug("No elements found with css. Validation failed.", css=css_selector, element_id=element_id)
raise MissingElement(selector=css_selector, element_id=element_id)
@@ -542,7 +545,7 @@ class SkyvernElement:
return await self.get_selectable() or self.get_tag_name() in SELECTABLE_ELEMENT
async def is_visible(self, must_visible_style: bool = True) -> bool:
- if not await self.get_locator().count():
+ if not await bounded_lookup_call(self.get_locator().count):
return False
if not must_visible_style:
return True
@@ -676,8 +679,10 @@ class SkyvernElement:
parent_locator = hover_target.locator("..")
try:
- if await parent_locator.count() != 1:
+ if await bounded_lookup_call(parent_locator.count) != 1:
break
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
LOG.debug(
"Unable to evaluate parent locator during hover reveal", exc_info=True, element_id=self.get_id()
@@ -975,7 +980,7 @@ class SkyvernElement:
locator = self.get_frame().locator(f"[id='{for_id}']")
# supposed to be only one element, since id is unique in the whole DOM
- if await locator.count() != 1:
+ if await bounded_lookup_call(locator.count) != 1:
return None
unique_id = await locator.get_attribute(SKYVERN_ID_ATTR, timeout=timeout)
@@ -997,7 +1002,9 @@ class SkyvernElement:
# fallback: fail_closed=True treats the label as forwarding so it is refused
# rather than clicked unvetted.
try:
- return await label_locator.locator("a[href], button").count() > 0
+ return await bounded_lookup_call(label_locator.locator("a[href], button").count) > 0
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
return fail_closed
@@ -1016,7 +1023,9 @@ class SkyvernElement:
if self.get_tag_name() != "label":
return True
try:
- return await locator.locator("a[href], button").count() == 0
+ return await bounded_lookup_call(locator.locator("a[href], button").count) == 0
+ except SkyvernPageAnalysisTimeout:
+ raise
except Exception:
return False
@@ -1029,7 +1038,7 @@ class SkyvernElement:
return None
locator = self.get_frame().locator(f"label[for='{element_id}']")
- cnt = await locator.count()
+ cnt = await bounded_lookup_call(locator.count)
if cnt == 1:
if await self._label_click_forwards_to_descendant(locator):
return None
@@ -1044,7 +1053,7 @@ class SkyvernElement:
return None
parent_locator = self.get_locator().locator("..")
- cnt = await parent_locator.count()
+ cnt = await bounded_lookup_call(parent_locator.count)
if cnt != 1:
return None
@@ -1124,7 +1133,7 @@ class SkyvernElement:
async def find_file_input_in_children(self) -> Locator | None:
"""Sometime the file input is invisible on the page, so it won't exist in the element tree, but it can be found in the DOM."""
locator = self.get_locator().locator('input[type="file"]')
- if await locator.count() != 1:
+ if await bounded_lookup_call(locator.count) != 1:
return None
return locator
@@ -1181,7 +1190,7 @@ class SkyvernElement:
# count() itself raises when the locator's frame detached; that means 0 matches
# here, not a new error to surface.
try:
- return await locator.count()
+ return await bounded_lookup_call(locator.count)
except Exception as exc:
if is_engine_error(exc, self._engine_selection) and is_element_detached_error(exc):
return 0
@@ -1302,7 +1311,7 @@ class SkyvernElement:
)
wait_config = await get_or_create_wait_config(task_id, workflow_run_id, organization_id)
await asyncio.sleep(get_wait_time(wait_config, "checkbox_retry_delay", default=2.0))
- if await self.get_locator().count() == 0:
+ if await bounded_lookup_call(self.get_locator().count) == 0:
LOG.info("Element is not on the page, the checking should work", element_id=self.get_id())
return
await self.get_locator().check(timeout=timeout)
@@ -1326,7 +1335,7 @@ class SkyvernElement:
)
wait_config = await get_or_create_wait_config(task_id, workflow_run_id, organization_id)
await asyncio.sleep(get_wait_time(wait_config, "checkbox_retry_delay", default=2.0))
- if await self.get_locator().count() == 0:
+ if await bounded_lookup_call(self.get_locator().count) == 0:
LOG.info("Element is not on the page, the unchecking should work", element_id=self.get_id())
return
await self.get_locator().uncheck(timeout=timeout)
@@ -1750,7 +1759,7 @@ class DomUtil:
self.scraped_page, self.page, frame, css, engine_selection=self.engine_selection
)
- num_elements = await locator.count()
+ num_elements = await bounded_lookup_call(locator.count)
if num_elements < 1:
if not allow_xpath_fallback:
raise MissingElement(selector=css, element_id=element_id)
@@ -1765,7 +1774,7 @@ class DomUtil:
# (maybe) we should validate the element hash to make sure the element is the same?
LOG.info("Fallback to locator element by xpath.", xpath=xpath, element_id=element_id, sampling=True)
locator = frame_content.locator(f"xpath={xpath}")
- num_elements = await locator.count()
+ num_elements = await bounded_lookup_call(locator.count)
if num_elements < 1:
raise MissingElement(selector=xpath, element_id=element_id)
# The tag-name xpath can resolve to many repeated nodes (rich text editors,
@@ -1796,6 +1805,11 @@ class DomUtil:
async def safe_get_skyvern_element_by_id(self, element_id: str) -> SkyvernElement | None:
try:
return await self.get_skyvern_element_by_id(element_id)
+ except SkyvernPageAnalysisTimeout:
+ # An unresponsive page is not an absent element: answering None would send the caller on
+ # to more driver calls against a renderer that stopped answering. Fail closed instead so
+ # the step ends and the next scrape's reload rung gets a chance at the page.
+ raise
except Exception:
LOG.warning("Failed to get skyvern element by id", element_id=element_id, exc_info=True)
return None
@@ -1846,7 +1860,7 @@ class DomUtil:
if not isinstance(css, str) or not css:
continue
locator = element.get_frame().locator(css)
- if await locator.count() != 1:
+ if await bounded_lookup_call(locator.count) != 1:
continue
owner = SkyvernElement(
locator,
diff --git a/tests/unit/test_auto_completion_discover_fallback.py b/tests/unit/test_auto_completion_discover_fallback.py
index ec953b381..548ef93dd 100644
--- a/tests/unit/test_auto_completion_discover_fallback.py
+++ b/tests/unit/test_auto_completion_discover_fallback.py
@@ -15,14 +15,16 @@ import pytest
from skyvern.exceptions import NoIncrementalElementFoundForAutoCompletion, NoSuitableAutoCompleteOption
from skyvern.forge.sdk.models import StepStatus
-from skyvern.webeye.actions.actions import InputOrSelectContext
+from skyvern.webeye.actions.actions import InputOrSelectContext, InputTextAction
from skyvern.webeye.actions.handler import (
AutoCompletionResult,
choose_auto_completion_dropdown,
discover_and_select_from_full_dropdown,
+ handle_input_text_action,
input_or_auto_complete_input,
)
from skyvern.webeye.actions.responses import ActionFailure, ActionSuccess
+from tests.unit.conftest import make_input_element_mock
from tests.unit.helpers import make_organization, make_step, make_task
# ---------------------------------------------------------------------------
@@ -934,3 +936,310 @@ async def test_input_or_auto_complete_skips_discover_for_search_bar() -> None:
# Search bar returns None early, never reaching discover fallback
assert result is None
mock_discover.assert_not_called()
+
+
+# ---------------------------------------------------------------------------
+# Wall-clock budget on the guessed-value fanout
+# ---------------------------------------------------------------------------
+
+
+def _budget_attempt_mock(seconds_per_attempt: float, elapsed: dict[str, float]) -> AsyncMock:
+ """A probe that advances a fake clock, so tests do not depend on how often it is read."""
+
+ async def _attempt(*_args: object, **_kwargs: object) -> AutoCompletionResult:
+ elapsed["seconds"] += seconds_per_attempt
+ return AutoCompletionResult(action_result=ActionFailure(exception=Exception("no match")))
+
+ return AsyncMock(side_effect=_attempt)
+
+
+async def test_ladder_stops_guessing_once_the_retry_budget_is_spent() -> None:
+ """The guessed-value fanout is bounded by wall clock, and discovery still runs.
+
+ Each guess costs a full retype plus a DOM scrape; production attempts reached ~62s apiece, so
+ three of them spent over four minutes on one field before discovery was even tried.
+ """
+ skyvern_el = _mock_skyvern_element()
+ elapsed = {"seconds": 0.0}
+ attempt = _budget_attempt_mock(40.0, elapsed)
+
+ with (
+ patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=attempt),
+ patch("skyvern.webeye.actions.handler.app") as mock_app,
+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
+ patch("skyvern.webeye.actions.handler.settings.AUTO_COMPLETION_LADDER_BUDGET_SECONDS", 60.0),
+ patch(
+ "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
+ new=AsyncMock(return_value=ActionSuccess()),
+ ) as mock_discover,
+ ):
+ mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(
+ return_value={
+ "potential_values": [
+ {"value": "New York County"},
+ {"value": "Kings County"},
+ {"value": "Queens County"},
+ ]
+ }
+ )
+ mock_prompt.load_prompt.return_value = "mocked prompt"
+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
+
+ result = await input_or_auto_complete_input(
+ input_or_select_context=_make_context(field="County"),
+ scraped_page=MagicMock(),
+ page=MagicMock(),
+ dom=MagicMock(),
+ text="New York",
+ skyvern_element=skyvern_el,
+ step=_STEP,
+ task=_TASK,
+ is_secret_value=False,
+ monotonic=lambda: elapsed["seconds"],
+ )
+
+ # Probe (40s) stays under budget so one guess is tried (80s); the remaining two are dropped.
+ assert attempt.await_count == 2
+ assert isinstance(result, ActionSuccess)
+ mock_discover.assert_awaited_once()
+
+
+async def test_ladder_skips_the_fanout_when_the_probe_alone_spends_the_budget() -> None:
+ """A probe that already overran the budget must not buy LLM guesses it cannot try."""
+ skyvern_el = _mock_skyvern_element()
+ elapsed = {"seconds": 0.0}
+ attempt = _budget_attempt_mock(70.0, elapsed)
+
+ with (
+ patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=attempt),
+ patch("skyvern.webeye.actions.handler.app") as mock_app,
+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
+ patch("skyvern.webeye.actions.handler.settings.AUTO_COMPLETION_LADDER_BUDGET_SECONDS", 60.0),
+ patch(
+ "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
+ new=AsyncMock(return_value=ActionSuccess()),
+ ) as mock_discover,
+ ):
+ mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(return_value={"potential_values": [{"value": "Kings County"}]})
+ mock_prompt.load_prompt.return_value = "mocked prompt"
+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
+
+ result = await input_or_auto_complete_input(
+ input_or_select_context=_make_context(field="County"),
+ scraped_page=MagicMock(),
+ page=MagicMock(),
+ dom=MagicMock(),
+ text="New York",
+ skyvern_element=skyvern_el,
+ step=_STEP,
+ task=_TASK,
+ is_secret_value=False,
+ monotonic=lambda: elapsed["seconds"],
+ )
+
+ assert attempt.await_count == 1
+ mock_app.SECONDARY_LLM_API_HANDLER.assert_not_called()
+ assert isinstance(result, ActionSuccess)
+ mock_discover.assert_awaited_once()
+
+
+async def test_ladder_tries_every_guess_when_attempts_are_fast() -> None:
+ """A responsive control keeps the full fanout: the budget must not truncate healthy ladders."""
+ skyvern_el = _mock_skyvern_element()
+ elapsed = {"seconds": 0.0}
+ attempt = _budget_attempt_mock(2.0, elapsed)
+
+ with (
+ patch("skyvern.webeye.actions.handler.choose_auto_completion_dropdown", new=attempt),
+ patch("skyvern.webeye.actions.handler.app") as mock_app,
+ patch("skyvern.webeye.actions.handler.prompt_engine") as mock_prompt,
+ patch("skyvern.webeye.actions.handler.skyvern_context") as mock_ctx,
+ patch("skyvern.webeye.actions.handler.settings.AUTO_COMPLETION_LADDER_BUDGET_SECONDS", 60.0),
+ patch(
+ "skyvern.webeye.actions.handler.discover_and_select_from_full_dropdown",
+ new=AsyncMock(return_value=None),
+ ),
+ ):
+ mock_app.SECONDARY_LLM_API_HANDLER = AsyncMock(
+ return_value={
+ "potential_values": [
+ {"value": "New York County"},
+ {"value": "Kings County"},
+ {"value": "Queens County"},
+ ]
+ }
+ )
+ mock_prompt.load_prompt.return_value = "mocked prompt"
+ mock_ctx.ensure_context.return_value = MagicMock(tz_info=UTC)
+
+ result = await input_or_auto_complete_input(
+ input_or_select_context=_make_context(field="County"),
+ scraped_page=MagicMock(),
+ page=MagicMock(),
+ dom=MagicMock(),
+ text="New York",
+ skyvern_element=skyvern_el,
+ step=_STEP,
+ task=_TASK,
+ is_secret_value=False,
+ monotonic=lambda: elapsed["seconds"],
+ )
+
+ assert attempt.await_count == 4 # the probe plus all three guesses
+ assert result is None
+
+
+# ---------------------------------------------------------------------------
+# An exhausted ladder must not report success over an empty field
+# ---------------------------------------------------------------------------
+
+
+async def _run_exhausted_ladder_input(
+ *,
+ read_back: str | None,
+ is_search_bar: bool = False,
+ incremental_options: list[dict] | None = None,
+ committed_selection: bool = False,
+ secret_value: str | None = None,
+ selection_readback: AsyncMock | None = None,
+) -> tuple[list, MagicMock, AsyncMock]:
+ """Drive handle_input_text_action with a ladder that selected nothing.
+
+ ``read_back`` is what the field holds once the ladder has given up and the Tab commit has
+ fired -- the only signal separating a value that landed from a control that cleared itself
+ on blur.
+ """
+ skyvern_el = make_input_element_mock(element_id="AADC")
+ skyvern_el.is_auto_completion_input = AsyncMock(return_value=True)
+ dom_instance = MagicMock()
+ dom_instance.get_skyvern_element_by_id = AsyncMock(return_value=skyvern_el)
+
+ inc = MagicMock()
+ inc.start_listen_dom_increment = AsyncMock()
+ inc.stop_listen_dom_increment = AsyncMock()
+ # Non-empty so the Tab commit hack arms, as it does on a real autocomplete control.
+ inc.get_incremental_element_tree = AsyncMock(
+ return_value=incremental_options if incremental_options is not None else [{"id": "OPT", "text": "Nassau"}]
+ )
+
+ skyvern_frame = MagicMock()
+ skyvern_frame.safe_wait_for_animation_end = AsyncMock()
+
+ scraped_page = MagicMock()
+ scraped_page.id_to_element_dict = {"AADC": {"tagName": "input"}}
+
+ context = InputOrSelectContext(field="County", is_search_bar=is_search_bar, is_location_input=False)
+ action_text = "{{login.username}}" if secret_value is not None else "New York"
+ resolved_text = secret_value if secret_value is not None else "New York"
+ action = InputTextAction(element_id="AADC", text=action_text, reasoning="fill the county")
+ selection_readback = selection_readback or AsyncMock(
+ return_value=(committed_selection, "scope_token" if committed_selection else "none")
+ )
+
+ with (
+ patch("skyvern.webeye.actions.handler.DomUtil", return_value=dom_instance),
+ patch("skyvern.webeye.actions.handler.SkyvernFrame.create_instance", new=AsyncMock(return_value=skyvern_frame)),
+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc),
+ patch("skyvern.webeye.actions.handler.get_input_value", new=AsyncMock(return_value=read_back)),
+ patch(
+ "skyvern.webeye.actions.handler.get_actual_value_of_parameter_if_secret_with_task",
+ return_value=resolved_text,
+ ),
+ patch("skyvern.webeye.actions.handler._get_input_or_select_context", new=AsyncMock(return_value=context)),
+ patch(
+ "skyvern.webeye.actions.handler._is_collapse_autocomplete_fanout_enabled",
+ new=AsyncMock(return_value=False),
+ ),
+ patch("skyvern.webeye.actions.handler._is_combobox_or_typeahead", new=AsyncMock(return_value=False)),
+ patch("skyvern.webeye.actions.handler._fill_secret_with_readback", new=AsyncMock(return_value=None)),
+ # No in-action custom selection commits, so the ladder is the only thing that could have.
+ patch("skyvern.webeye.actions.handler.sequentially_select_from_dropdown", new=AsyncMock(return_value=None)),
+ patch(
+ "skyvern.webeye.actions.handler._custom_select_scope_confirms_committed",
+ new=selection_readback,
+ ),
+ # The ladder ran every attempt and selected nothing.
+ patch("skyvern.webeye.actions.handler.input_or_auto_complete_input", new=AsyncMock(return_value=None)),
+ ):
+ results = await handle_input_text_action(
+ action=action, page=MagicMock(), scraped_page=scraped_page, task=_TASK, step=_STEP
+ )
+ return results, skyvern_el, selection_readback
+
+
+async def test_exhausted_ladder_over_an_empty_field_fails_the_action() -> None:
+ """The ladder selected nothing and the field is empty -> a failed action, not 'Action succeeded'.
+
+ Reporting success here hides a blank required field from the agent, which then keeps filling
+ the rest of the form and submits it.
+ """
+ results, _, _ = await _run_exhausted_ladder_input(read_back="")
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionFailure)
+ assert results[0].exception_type == "NoSuitableAutoCompleteOption"
+ assert results[0].stop_execution_on_failure is True
+ assert results[0].skip_remaining_actions is True
+ assert "no matching option was committed" in (results[0].exception_message or "")
+ assert "field was left empty" not in (results[0].exception_message or "")
+
+
+async def test_exhausted_ladder_that_left_a_value_still_succeeds() -> None:
+ """The typed text survived the commit, so the field is filled and the action succeeded."""
+ results, _, _ = await _run_exhausted_ladder_input(read_back="New York")
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+
+
+async def test_exhausted_ladder_on_a_search_bar_still_succeeds() -> None:
+ """A search bar legitimately holds typed text with nothing selected, so it is not armed."""
+ results, _, _ = await _run_exhausted_ladder_input(read_back="", is_search_bar=True)
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+
+
+async def test_exhausted_ladder_blurs_even_when_no_options_were_detected() -> None:
+ results, skyvern_el, _ = await _run_exhausted_ladder_input(read_back="", incremental_options=[])
+
+ assert isinstance(results[0], ActionFailure)
+ assert ("Tab",) in [call.args for call in skyvern_el.press_key.await_args_list]
+
+
+async def test_exhausted_ladder_treats_a_committed_chip_as_filled() -> None:
+ results, _, selection_readback = await _run_exhausted_ladder_input(read_back="", committed_selection=True)
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
+ assert selection_readback.await_args.kwargs["expected_label"] == "new york"
+ assert selection_readback.await_args.kwargs["allow_aria_selected_option_tokens"] is False
+
+
+async def test_exhausted_ladder_redacts_secret_from_failure_message() -> None:
+ secret_value = "unlogged-sensitive-value-9e2d"
+ results, _, selection_readback = await _run_exhausted_ladder_input(read_back="", secret_value=secret_value)
+
+ assert isinstance(results[0], ActionFailure)
+ assert secret_value not in (results[0].exception_message or "")
+ assert "target_value=<secret>" in (results[0].exception_message or "")
+ assert selection_readback.await_args.kwargs["matched_label"] == "<secret>"
+
+
+async def test_exhausted_ladder_does_not_treat_an_unreadable_field_as_empty() -> None:
+ results, _, _ = await _run_exhausted_ladder_input(read_back=None)
+
+ assert len(results) == 1
+ assert isinstance(results[0], ActionSuccess)
diff --git a/tests/unit/test_auto_completion_location.py b/tests/unit/test_auto_completion_location.py
index b3b985afb..76e0a317c 100644
--- a/tests/unit/test_auto_completion_location.py
+++ b/tests/unit/test_auto_completion_location.py
@@ -13,7 +13,11 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from skyvern.constants import SKYVERN_ID_ATTR
-from skyvern.exceptions import InvalidElementForTextInput, NoIncrementalElementFoundForAutoCompletion
+from skyvern.exceptions import (
+ InvalidElementForTextInput,
+ NoIncrementalElementFoundForAutoCompletion,
+ SkyvernPageAnalysisTimeout,
+)
from skyvern.forge.sdk.models import StepStatus
from skyvern.webeye.actions.actions import InputOrSelectContext
from skyvern.webeye.actions.handler import (
@@ -144,6 +148,36 @@ async def test_location_single_option_skips_llm() -> None:
assert isinstance(result.action_result, ActionSuccess)
+async def test_auto_completion_propagates_page_analysis_timeout() -> None:
+ frame = _mock_frame()
+ frame.locator.return_value.count = AsyncMock(side_effect=SkyvernPageAnalysisTimeout("lookup timed out"))
+ skyvern_el = _mock_skyvern_element(frame)
+ inc_scrape = _mock_incremental_scrape(SINGLE_ELEMENT)
+
+ with (
+ patch(
+ "skyvern.webeye.actions.handler.SkyvernFrame.create_instance",
+ new=AsyncMock(return_value=MagicMock(safe_wait_for_animation_end=AsyncMock())),
+ ),
+ patch("skyvern.webeye.actions.handler.IncrementalScrapePage", return_value=inc_scrape),
+ ):
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ await choose_auto_completion_dropdown(
+ context=_make_location_context(),
+ page=MagicMock(),
+ scraped_page=MagicMock(),
+ dom=MagicMock(),
+ text="123 Main St",
+ skyvern_element=skyvern_el,
+ step=_STEP,
+ task=_TASK,
+ preserved_elements=[{"id": "preserved"}],
+ is_secret_value=False,
+ is_location_input=True,
+ )
+
+
@pytest.mark.asyncio
async def test_location_whitespace_normalized_still_matches() -> None:
"""Input with extra whitespace should still match after normalization."""
diff --git a/tests/unit/test_click_desired_state.py b/tests/unit/test_click_desired_state.py
index bc5a92dcd..fca174892 100644
--- a/tests/unit/test_click_desired_state.py
+++ b/tests/unit/test_click_desired_state.py
@@ -21,6 +21,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from playwright.async_api import Page, async_playwright
+from skyvern.exceptions import SkyvernPageAnalysisTimeout
from skyvern.webeye.actions import handler
from skyvern.webeye.actions.actions import ActionStatus, ClickAction, ClickContext
from skyvern.webeye.actions.responses import ActionAbort, ActionFailure, ActionResult, ActionSuccess, StaleActionAbort
@@ -499,6 +500,15 @@ async def test_resolve_read_exception_returns_none() -> None:
assert await handler._resolve_live_selected_state(element) is None
+async def test_apply_desired_state_propagates_selected_option_lookup_timeout() -> None:
+ element = FakeToggleElement("div", {"role": "option", "aria-selected": "true"})
+ timeout = SkyvernPageAnalysisTimeout("lookup timed out")
+ with patch.object(handler, "bounded_lookup_call", AsyncMock(side_effect=timeout)):
+ with pytest.raises(SkyvernPageAnalysisTimeout, match="lookup timed out"):
+ await handler._apply_desired_click_state(_click_action(), element, True, MagicMock())
+
+
@pytest.mark.parametrize("attributes", [{}, {"role": "button"}, {"href": "https://example.test/next"}])
@pytest.mark.asyncio
async def test_resolve_control_without_boolean_observable_returns_none(attributes: dict[str, str]) -> None:
@@ -1348,6 +1358,29 @@ def _static_listbox_snapshot(**overrides: object) -> dict[str, object]:
return snapshot
+async def test_static_listbox_resolution_propagates_lookup_timeout() -> None:
+ timeout = SkyvernPageAnalysisTimeout("lookup timed out")
+ option = MagicMock()
+ option.is_visible = AsyncMock(return_value=True)
+ option.element_handle = AsyncMock(return_value=MagicMock())
+ option.get_attribute = AsyncMock(return_value="option-id")
+ element = MagicMock()
+ element.get_locator.return_value.locator.return_value.all = AsyncMock(return_value=[option])
+ element.get_frame.return_value = MagicMock()
+ element.get_id.return_value = "listbox"
+ dom = MagicMock()
+ dom.safe_get_skyvern_element_by_id = AsyncMock(side_effect=timeout)
+
+ with patch.object(
+ handler, "read_static_listbox_option_snapshot", new=AsyncMock(return_value=_static_listbox_snapshot())
+ ):
+ with pytest.raises(SkyvernPageAnalysisTimeout, match="lookup timed out"):
+ await handler._resolve_static_listbox_container_target(_click_action(), element, dom)
+
+ dom.safe_get_skyvern_element_by_id.assert_awaited_once_with("option-id")
+
+
@pytest.mark.parametrize(
("snapshot", "expected"),
[
@@ -1978,6 +2011,17 @@ async def test_apply_grid_row_driver_failure_falls_open() -> None:
drive.assert_awaited_once()
+async def test_grid_row_driver_propagates_page_analysis_timeout() -> None:
+ element = MagicMock()
+ cell_locator = MagicMock()
+ cell_locator.count = AsyncMock(side_effect=SkyvernPageAnalysisTimeout("lookup timed out"))
+ element.get_locator.return_value.locator.return_value.first = cell_locator
+
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ await handler._drive_grid_row_selection(_click_action(), element, True)
+
+
@pytest.mark.asyncio
async def test_apply_grid_row_unmarked_desired_true_falls_open_without_driver() -> None:
# v3.1 invariant: an UNMARKED row (no positive selected signal) with desired=True is DOM-
diff --git a/tests/unit/test_incremental_scrape_lookup_deadline.py b/tests/unit/test_incremental_scrape_lookup_deadline.py
new file mode 100644
index 000000000..602555482
--- /dev/null
+++ b/tests/unit/test_incremental_scrape_lookup_deadline.py
@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from collections.abc import Awaitable, Callable
+from types import SimpleNamespace
+
+import pytest
+
+from skyvern.exceptions import SkyvernPageAnalysisTimeout
+from skyvern.webeye.scraper import scraper as scraper_module
+from skyvern.webeye.scraper.scraper import IncrementalScrapePage
+
+
+class _CountLocator:
+ def __init__(self, selector: str) -> None:
+ self.selector = selector
+
+ async def count(self) -> int:
+ return 1
+
+
+class _Frame:
+ def locator(self, selector: str) -> _CountLocator:
+ return _CountLocator(selector)
+
+
[email protected]("timed_out_element_id", ["child", "parent"])
+async def test_select_one_element_by_value_propagates_lookup_deadline(
+ monkeypatch: pytest.MonkeyPatch, timed_out_element_id: str
+) -> None:
+ page = IncrementalScrapePage(SimpleNamespace(get_frame=lambda: _Frame())) # type: ignore[arg-type]
+ page.element_tree = [
+ {
+ "id": "parent",
+ "interactable": True,
+ "children": [{"id": "child", "text": "target", "interactable": False}],
+ }
+ ]
+ lookups: list[str] = []
+
+ async def bounded_lookup(call: Callable[[], Awaitable[int]]) -> int:
+ element_id = "child" if '"child"' in call.__self__.selector else "parent"
+ lookups.append(element_id)
+ if element_id == timed_out_element_id:
+ raise SkyvernPageAnalysisTimeout("lookup timed out")
+ return await call()
+
+ monkeypatch.setattr(scraper_module, "bounded_lookup_call", bounded_lookup)
+
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ await page.select_one_element_by_value("target")
+
+ assert lookups[-1] == timed_out_element_id
diff --git a/tests/unit/test_multi_field_totp.py b/tests/unit/test_multi_field_totp.py
index 490eae1b8..5e23d45e4 100644
--- a/tests/unit/test_multi_field_totp.py
+++ b/tests/unit/test_multi_field_totp.py
@@ -23,6 +23,7 @@ from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from pydantic import TypeAdapter
from skyvern.config import settings
+from skyvern.exceptions import SkyvernPageAnalysisTimeout
from skyvern.forge import agent as agent_module
from skyvern.forge.agent import (
ForgeAgent,
@@ -4500,6 +4501,44 @@ async def test_submission_evidence_filters_volatile_nodes_and_rebinds_scopes(
assert rebound.evaluate.await_args is not None
+async def test_group_binding_propagates_page_analysis_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
+ attempt = _attempt()
+ group = SimpleNamespace(
+ box_element_ids=attempt.box_element_ids,
+ frame="main.frame",
+ container={"id": "otp-container"},
+ )
+ fresh = SimpleNamespace(
+ url="same",
+ _document_loader_id="loader",
+ id_to_frame_dict={element_id: "main.frame" for element_id in attempt.box_element_ids},
+ )
+ original = SimpleNamespace(
+ url="same",
+ _document_loader_id="loader",
+ id_to_frame_dict={element_id: "main.frame" for element_id in attempt.box_element_ids},
+ id_to_css_dict={element_id: f"#{element_id}" for element_id in attempt.box_element_ids},
+ generate_scraped_page_without_screenshots=AsyncMock(return_value=fresh),
+ )
+ page = SimpleNamespace(url="same")
+ timeout = SkyvernPageAnalysisTimeout("lookup timed out")
+ lookup = AsyncMock(side_effect=timeout)
+ locator = SimpleNamespace(count=AsyncMock(return_value=1))
+
+ monkeypatch.setattr(multi_field_totp_module, "_document_continuity", AsyncMock(return_value=True))
+ monkeypatch.setattr(multi_field_totp_module, "_multi_field_totp_box_groups", lambda *_: [group])
+ monkeypatch.setattr(multi_field_totp_module, "_multi_field_totp_container_identity", lambda *_: "same-widget")
+ monkeypatch.setattr(multi_field_totp_module, "_multi_field_totp_matching_scopes", lambda *_: [object()])
+ monkeypatch.setattr(multi_field_totp_module, "resolve_locator", AsyncMock(return_value=(locator, None)))
+ monkeypatch.setattr(multi_field_totp_module, "bounded_lookup_call", lookup)
+
+ with pytest.raises(SkyvernPageAnalysisTimeout, match="lookup timed out"):
+ await multi_field_totp_module._refresh_multi_field_totp_group_binding(original, page, attempt)
+
+ lookup.assert_awaited_once()
+
+
def test_widget_probe_hashes_own_nodes_without_secret_values_or_descendant_churn() -> None:
node = shutil.which("node")
if node is None:
@@ -4720,6 +4759,16 @@ async def test_fill_observer_preserves_only_counts_across_document_teardown() ->
assert state.observed_max_filled == 0
+async def test_fill_observer_refresh_propagates_page_analysis_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
+ timeout = SkyvernPageAnalysisTimeout("lookup timed out")
+ monkeypatch.setattr(multi_field_totp_module, "bounded_lookup_call", AsyncMock(side_effect=timeout))
+ observer = multi_field_totp_module.MultiFieldTotpFillObserver(SimpleNamespace(), _attempt(), "test:")
+
+ with pytest.raises(SkyvernPageAnalysisTimeout, match="lookup timed out"):
+ await observer.refresh([SimpleNamespace(count=AsyncMock())])
+
+
def test_input_observer_measures_peak_fullness_before_site_clear() -> None:
for clear_at in (2, 4):
observed = _probe_live_fill_observer(remount=False, clear_at=clear_at)
diff --git a/tests/unit/test_stale_web_action_remap.py b/tests/unit/test_stale_web_action_remap.py
index 966cc8a2b..022aaf855 100644
--- a/tests/unit/test_stale_web_action_remap.py
+++ b/tests/unit/test_stale_web_action_remap.py
@@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
+from skyvern.exceptions import SkyvernPageAnalysisTimeout
from skyvern.webeye.actions import handler
from skyvern.webeye.actions.actions import ClickAction
@@ -381,3 +382,25 @@ async def test_indeterminate_probe_falls_through_to_legacy(monkeypatch: pytest.M
assert result is None
assert action.element_id == "OLD"
rescrape.assert_not_awaited()
+
+
+async def test_stale_remap_propagates_lookup_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ batch = _scraped({"OLD": _element("Approve", html_id="approve_field")})
+ monkeypatch.setattr(handler, "_document_continuity", AsyncMock(return_value=True))
+ locator = MagicMock()
+ locator.count = AsyncMock(side_effect=SkyvernPageAnalysisTimeout("lookup deadline"))
+ monkeypatch.setattr(handler, "resolve_locator", AsyncMock(return_value=(locator, MagicMock())))
+
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ await handler._refresh_stale_web_action_before_dispatch(batch, _live_page(), ClickAction(element_id="OLD"))
+
+
+async def test_stale_batch_probe_propagates_lookup_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ batch = _scraped({"OLD": _element("Approve", html_id="approve_field")})
+ monkeypatch.setattr(handler, "_document_continuity", AsyncMock(return_value=True))
+ locator = MagicMock()
+ locator.count = AsyncMock(side_effect=SkyvernPageAnalysisTimeout("lookup deadline"))
+ monkeypatch.setattr(handler, "resolve_locator", AsyncMock(return_value=(locator, MagicMock())))
+
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ await handler._batched_target_stale_beyond_remap(batch, _live_page(), ClickAction(element_id="OLD"))
diff --git a/tests/unit/test_xpath_fallback_ambiguity.py b/tests/unit/test_xpath_fallback_ambiguity.py
index 993d14aef..99f3a5feb 100644
--- a/tests/unit/test_xpath_fallback_ambiguity.py
+++ b/tests/unit/test_xpath_fallback_ambiguity.py
@@ -1,10 +1,13 @@
from __future__ import annotations
+import asyncio
from unittest.mock import AsyncMock, MagicMock
import pytest
-from skyvern.exceptions import MissingElement, MultipleElementsFound
+from skyvern.config import settings
+from skyvern.exceptions import MissingElement, MultipleElementsFound, SkyvernPageAnalysisTimeout
+from skyvern.webeye import action_deadline
from skyvern.webeye.utils.dom import DomUtil
@@ -18,7 +21,7 @@ def _make_dom() -> DomUtil:
return DomUtil(scraped_page, MagicMock())
-def _patch_resolve(monkeypatch: pytest.MonkeyPatch, *, css_count: int, xpath_count: int) -> MagicMock:
+def _patch_resolve(monkeypatch: pytest.MonkeyPatch, *, css_count: int, xpath_count: int) -> tuple[MagicMock, MagicMock]:
css_locator = MagicMock()
css_locator.count = AsyncMock(return_value=css_count)
xpath_locator = MagicMock()
@@ -32,7 +35,7 @@ def _patch_resolve(monkeypatch: pytest.MonkeyPatch, *, css_count: int, xpath_cou
return css_locator, frame_content
monkeypatch.setattr("skyvern.webeye.utils.dom.resolve_locator", _resolve)
- return xpath_locator
+ return css_locator, xpath_locator
@pytest.mark.asyncio
@@ -61,3 +64,42 @@ async def test_css_multiple_matches_still_rejected(monkeypatch: pytest.MonkeyPat
_patch_resolve(monkeypatch, css_count=2, xpath_count=1)
with pytest.raises(MultipleElementsFound):
await _make_dom().get_skyvern_element_by_id("el-1")
+
+
+async def test_lookup_against_a_frozen_page_ends_at_its_deadline(monkeypatch: pytest.MonkeyPatch) -> None:
+ """``count()`` takes no Playwright timeout, so a renderer that stopped answering held one action
+ for 15.5 minutes until the page recovered on its own."""
+ monkeypatch.setattr(action_deadline, "ACTION_DEADLINE_HEADROOM_MS", 0)
+ monkeypatch.setattr(settings, "BROWSER_ELEMENT_LOOKUP_TIMEOUT_MS", 50)
+ css_locator, _ = _patch_resolve(monkeypatch, css_count=0, xpath_count=1)
+ frozen = asyncio.Event()
+
+ async def never_answers() -> int:
+ await frozen.wait()
+ return 0
+
+ css_locator.count = never_answers
+
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ # A lookup that ignores its budget fails the test here instead of hanging the suite.
+ async with asyncio.timeout(10):
+ await _make_dom().get_skyvern_element_by_id("el-1")
+
+
+async def test_safe_lookup_propagates_analysis_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(action_deadline, "ACTION_DEADLINE_HEADROOM_MS", 0)
+ monkeypatch.setattr(settings, "BROWSER_ELEMENT_LOOKUP_TIMEOUT_MS", 50)
+ css_locator, _ = _patch_resolve(monkeypatch, css_count=0, xpath_count=1)
+ frozen = asyncio.Event()
+
+ async def never_answers() -> int:
+ await frozen.wait()
+ return 0
+
+ css_locator.count = never_answers
+
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ async with asyncio.timeout(10):
+ await _make_dom().safe_get_skyvern_element_by_id("el-1")