Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Avoid replaying ordinary JavaScript ReferenceErrors (SKY-15778) (#8489)
Andrew Neilsoneaf99422762e
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 1f57284f4..27528aeb0 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -45,7 +45,7 @@ from skyvern.webeye.scraper.scraped_page import (json_to_html,)from skyvern.webeye.utils.document import get_main_document_loader_id-from skyvern.webeye.utils.page import SkyvernFrame, load_js_script+from skyvern.webeye.utils.page import SkyvernFrame, load_js_script, with_dom_utilsif TYPE_CHECKING:from skyvern.webeye.browser_engine import BrowserEngineSelection@@ -989,7 +989,9 @@ class IncrementalScrapePage(ElementTreeBuilder):return self.element_tree_trimmedasync def start_listen_dom_increment(self, element: ElementHandle | None = None) -> None:- js_script = "async (element) => await startGlobalIncrementalObserver(element)"+ js_script = with_dom_utils(+ "async (element) => await startGlobalIncrementalObserver(element)", ("startGlobalIncrementalObserver",)+ )await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script, arg=element)async def stop_listen_dom_increment(self) -> None:@@ -997,7 +999,9 @@ class IncrementalScrapePage(ElementTreeBuilder):js_script = "() => window.globalObserverForDOMIncrement === undefined"if await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script):return- js_script = "async () => await stopGlobalIncrementalObserver()"+ js_script = with_dom_utils(+ "async () => await stopGlobalIncrementalObserver()", ("stopGlobalIncrementalObserver",)+ )await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(),expression=js_script,diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 10c4b77df..36ac29dbb 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -268,6 +268,33 @@ def load_js_script() -> str:JS_FUNCTION_DEFS = load_js_script()++class _DomUtilsExpression(str):+ """A guarded built-in expression whose helpers may be bootstrapped before it runs."""+++_DOM_UTILS_MISSING_RESULT_KEY = "__skyvern_dom_utils_missing__"+++def _is_dom_utils_missing_result(result: Any) -> bool:+ return result == {_DOM_UTILS_MISSING_RESULT_KEY: True}+++def with_dom_utils(expression: str, required_helpers: tuple[str, ...]) -> str:+ """Guard a built-in call so missing helpers are reported before the operation runs."""+ textual_enabled = SettingsManager.get_settings().ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE+ context = skyvern_context.current()+ enriched_enabled = bool(context and context.enriched_tree_enabled())+ return _DomUtilsExpression(f"""(arg) => {{+ if (!{json.dumps(required_helpers)}.every((name) => typeof globalThis[name] === "function")) {{+ return {{{json.dumps(_DOM_UTILS_MISSING_RESULT_KEY)}: true}};+ }}+ window.GlobalEnableAllTextualElements = {json.dumps(textual_enabled)};+ window.GlobalEnableEnrichedElementTree = {json.dumps(enriched_enabled)};+ return ({expression})(arg);+ }}""")++_NAVIGATION_RECOVERY_MAX_ATTEMPTS = 4_NAVIGATION_SETTLE_TIMEOUT_MS = 3000@@ -302,11 +329,7 @@ def _extract_playwright_screenshot_stage(exc: BaseException) -> str:def _is_navigation_context_lost(error_msg: str) -> bool:- if "Execution context was destroyed" in error_msg:- return True- if "Cannot find context with specified id" in error_msg:- return True- return "ReferenceError" in error_msg and "is not defined" in error_msg+ return "Execution context was destroyed" in error_msg or "Cannot find context with specified id" in error_msg# Order-locked labels for the selectors in LOADING_INDICATOR_PROBE_JS. The JS ships back a fixed@@ -608,6 +631,22 @@ async def _dispatch_evaluate(frame: Page | Frame, expression: str, arg: Any | Noreturn await evaluate_in_main_world(frame, expression, arg)+async def _dispatch_dom_utils_bootstrap(+ frame: Page | Frame, operation_arg: Any | None, *, force_cdp: bool = False+) -> Any:+ """Install helpers through the same world-selection route as the guarded operation."""+ if force_cdp:+ return await _dispatch_evaluate(frame, JS_FUNCTION_DEFS, None, force_cdp=True)+ if not is_page_like(frame):+ return await frame.evaluate(expression=JS_FUNCTION_DEFS, arg=None)+ context = frame.context+ if context is None or get_main_world_prefix(context) is None:+ return await frame.evaluate(expression=JS_FUNCTION_DEFS, arg=None)+ if operation_arg is not None and not _is_json_inlinable(operation_arg):+ return await frame.evaluate(expression=JS_FUNCTION_DEFS, arg=None)+ return await evaluate_in_main_world(frame, JS_FUNCTION_DEFS)++async def _wait_for_navigation_settle(frame: Page | Frame,timeout_ms: float,@@ -1501,12 +1540,16 @@ class SkyvernFrame:async def evaluate_expression() -> Any:return await _dispatch_evaluate(frame, expression, arg, force_cdp=force_cdp)+ async def bootstrap_expression() -> Any:+ return await _dispatch_dom_utils_bootstrap(frame, arg, force_cdp=force_cdp)+return await SkyvernFrame._evaluate_expression(frame=frame,expression=expression,evaluate_expression=evaluate_expression,timeout_ms=timeout_ms,engine_selection=engine_selection,+ bootstrap_expression=bootstrap_expression,**({"deadline": deadline} if deadline is not None else {}),)@@ -1518,7 +1561,23 @@ class SkyvernFrame:timeout_ms: float,engine_selection: BrowserEngineSelection | None = None,deadline: float | None = None,+ bootstrap_expression: Callable[[], Awaitable[Any]] | None = None,) -> Any:+ if isinstance(expression, _DomUtilsExpression):+ if bootstrap_expression is None:+ raise RuntimeError("domUtils expression requires a matching-world bootstrap")+ guarded_evaluate = evaluate_expression++ async def evaluate_expression() -> Any:+ for _ in range(_NAVIGATION_RECOVERY_MAX_ATTEMPTS):+ result = await guarded_evaluate()+ if not _is_dom_utils_missing_result(result):+ return result+ # The guarded expression returned before invoking the built-in operation, so+ # installing helpers and trying it again cannot repeat caller-visible effects.+ await bootstrap_expression()+ raise RuntimeError("domUtils helpers remained unavailable after bootstrap")+loop = asyncio.get_running_loop()deadline = deadline if deadline is not None else loop.time() + timeout_ms / 1000try:@@ -1528,7 +1587,7 @@ class SkyvernFrame:skyvern_context.record_browser_timeout(BrowserOperation.EVALUATE)# Re-raised and handled by the caller (scrape retries / failure classification),# so this is not the failure boundary; log without a traceback at warning.- LOG.warning("Skyvern timed out trying to analyze the page", expression=expression)+ LOG.warning("Skyvern timed out trying to analyze the page", expression=expression[:200])raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from errorexcept RuntimeError as e:# `evaluate_in_main_world` raises RuntimeError on Runtime.evaluate@@ -1594,7 +1653,7 @@ class SkyvernFrame:if _remaining_seconds() <= 0:LOG.warning("Skyvern timed out trying to analyze the page after navigation recovery",- expression=expression,+ expression=expression[:200],)raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page")@@ -1615,7 +1674,7 @@ class SkyvernFrame:if inject_budget <= 0:LOG.warning("Skyvern timed out trying to analyze the page after navigation recovery",- expression=expression,+ expression=expression[:200],)raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page")try:@@ -1626,7 +1685,7 @@ class SkyvernFrame:except asyncio.TimeoutError as error:LOG.warning("Skyvern timed out trying to analyze the page during domUtils.js re-injection",- expression=expression,+ expression=expression[:200],exc_info=True,)raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error@@ -1649,7 +1708,7 @@ class SkyvernFrame:if retry_budget <= 0:LOG.warning("Skyvern timed out trying to analyze the page after navigation recovery",- expression=expression,+ expression=expression[:200],)raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page")try:@@ -1660,7 +1719,9 @@ class SkyvernFrame:return resultexcept asyncio.TimeoutError as error:LOG.warning(- "Skyvern timed out on retry after JS context re-injection", expression=expression, exc_info=True+ "Skyvern timed out on retry after JS context re-injection",+ expression=expression[:200],+ exc_info=True,)raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from errorexcept Exception as retry_err:@@ -2032,7 +2093,7 @@ class SkyvernFrame:return await self.frame.content()async def get_scroll_x_y(self) -> tuple[int, int]:- js_script = "() => getScrollXY()"+ js_script = with_dom_utils("() => getScrollXY()", ("getScrollXY",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def get_open_aria_popup_trigger(self) -> dict | None:@@ -2043,7 +2104,10 @@ class SkyvernFrame:semantics live in getOpenAriaPopupTrigger in domUtils.js."""try:- result = await self.evaluate(frame=self.frame, expression="() => getOpenAriaPopupTrigger()")+ result = await self.evaluate(+ frame=self.frame,+ expression=with_dom_utils("() => getOpenAriaPopupTrigger()", ("getOpenAriaPopupTrigger",)),+ )except Exception:LOG.warning("Failed to detect open ARIA popup trigger; using default scrolling behavior",@@ -2053,11 +2117,11 @@ class SkyvernFrame:return result if isinstance(result, dict) else Noneasync def get_scroll_width_and_height(self) -> tuple[int, int]:- js_script = "() => getScrollWidthAndHeight()"+ js_script = with_dom_utils("() => getScrollWidthAndHeight()", ("getScrollWidthAndHeight",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def scroll_to_x_y(self, x: int, y: int) -> None:- js_script = "([x, y]) => scrollToXY(x, y)"+ js_script = with_dom_utils("([x, y]) => scrollToXY(x, y)", ("scrollToXY",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[x, y])@@ -2077,19 +2141,24 @@ class SkyvernFrame:)async def scroll_to_element_bottom(self, element: ElementHandle, page_by_page: bool = False) -> None:- js_script = "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)"+ js_script = with_dom_utils(+ "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)", ("scrollToElementBottom",)+ )return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element, page_by_page])async def scroll_to_element_top(self, element: ElementHandle) -> None:- js_script = "(element) => scrollToElementTop(element)"+ js_script = with_dom_utils("(element) => scrollToElementTop(element)", ("scrollToElementTop",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element)async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> dict:- js_script = "async ([frame, element, interactable]) => await buildElementObject(frame, element, interactable)"+ js_script = with_dom_utils(+ "async ([frame, element, interactable]) => await buildElementObject(frame, element, interactable)",+ ("buildElementObject",),+ )parsed = await self.evaluate(frame=self.frame,engine_selection=self.engine_selection,@@ -2100,35 +2169,41 @@ class SkyvernFrame:return parsedasync def get_element_scrollable(self, element: ElementHandle) -> bool:- js_script = "(element) => isScrollable(element)"+ js_script = with_dom_utils("(element) => isScrollable(element)", ("isScrollable",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element)async def get_element_visible(self, locator: Locator) -> bool:- js_script = "(element) => isElementVisible(element) && !isHidden(element)"+ js_script = with_dom_utils(+ "(element) => isElementVisible(element) && !isHidden(element)", ("isElementVisible", "isHidden")+ )async def evaluate_expression() -> bool:if await locator.count() == 0:return Falsereturn await locator.evaluate(js_script)+ async def bootstrap_expression() -> Any:+ return await locator.evaluate(f"(element) => {{ {JS_FUNCTION_DEFS} }}")+return await self._evaluate_expression(frame=self.frame,engine_selection=self.engine_selection,expression=js_script,evaluate_expression=evaluate_expression,+ bootstrap_expression=bootstrap_expression,timeout_ms=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,)async def get_disabled_from_style(self, element: ElementHandle) -> bool:- js_script = "(element) => checkDisabledFromStyle(element)"+ js_script = with_dom_utils("(element) => checkDisabledFromStyle(element)", ("checkDisabledFromStyle",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element)async def get_blocking_element_id(self, element: ElementHandle) -> tuple[str, bool]:- js_script = "(element) => getBlockElementUniqueID(element)"+ js_script = with_dom_utils("(element) => getBlockElementUniqueID(element)", ("getBlockElementUniqueID",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element)@@ -2140,7 +2215,10 @@ class SkyvernFrame::param page: Page instance to take the screenshot from.:return: Screenshot of the page."""- js_script = "async ([draw_boxes, frame, frame_index]) => await safeScrollToTop(draw_boxes, frame, frame_index)"+ js_script = with_dom_utils(+ "async ([draw_boxes, frame, frame_index]) => await safeScrollToTop(draw_boxes, frame, frame_index)",+ ("safeScrollToTop",),+ )scroll_y_px = await self.evaluate(frame=self.frame,engine_selection=self.engine_selection,@@ -2165,7 +2243,10 @@ class SkyvernFrame::param page: Page instance to take the screenshot from.:return: Screenshot of the page."""- js_script = "async ([draw_boxes, frame, frame_index, need_overlap]) => await scrollToNextPage(draw_boxes, frame, frame_index, need_overlap)"+ js_script = with_dom_utils(+ "async ([draw_boxes, frame, frame_index, need_overlap]) => await scrollToNextPage(draw_boxes, frame, frame_index, need_overlap)",+ ("scrollToNextPage",),+ )scroll_y_px = await self.evaluate(frame=self.frame,engine_selection=self.engine_selection,@@ -2186,7 +2267,7 @@ class SkyvernFrame:Remove the bounding boxes from the page.:param page: Page instance to remove the bounding boxes from."""- js_script = "() => removeBoundingBoxes()"+ js_script = with_dom_utils("() => removeBoundingBoxes()", ("removeBoundingBoxes",))await self.evaluate(frame=self.frame,engine_selection=self.engine_selection,@@ -2195,7 +2276,10 @@ class SkyvernFrame:)async def build_elements_and_draw_bounding_boxes(self, frame: str, frame_index: int) -> None:- js_script = "async ([frame, frame_index]) => await buildElementsAndDrawBoundingBoxes(frame, frame_index)"+ js_script = with_dom_utils(+ "async ([frame, frame_index]) => await buildElementsAndDrawBoundingBoxes(frame, frame_index)",+ ("buildElementsAndDrawBoundingBoxes",),+ )await self.evaluate(frame=self.frame,engine_selection=self.engine_selection,@@ -2205,23 +2289,23 @@ class SkyvernFrame:)async def is_window_scrollable(self) -> bool:- js_script = "() => isWindowScrollable()"+ js_script = with_dom_utils("() => isWindowScrollable()", ("isWindowScrollable",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def is_parent(self, parent: ElementHandle, child: ElementHandle) -> bool:- js_script = "([parent, child]) => isParent(parent, child)"+ js_script = with_dom_utils("([parent, child]) => isParent(parent, child)", ("isParent",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[parent, child])async def is_sibling(self, el1: ElementHandle, el2: ElementHandle) -> bool:- js_script = "([el1, el2]) => isSibling(el1, el2)"+ js_script = with_dom_utils("([el1, el2]) => isSibling(el1, el2)", ("isSibling",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[el1, el2])async def has_ASP_client_control(self) -> bool:- js_script = "() => hasASPClientControl()"+ js_script = with_dom_utils("() => hasASPClientControl()", ("hasASPClientControl",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def click_element_in_javascript(self, element: ElementHandle) -> None:@@ -2267,19 +2351,19 @@ class SkyvernFrame:)async def get_select_options(self, element: ElementHandle) -> tuple[list, str]:- js_script = "([element]) => getSelectOptions(element)"+ js_script = with_dom_utils("([element]) => getSelectOptions(element)", ("getSelectOptions",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element])async def get_element_dom_depth(self, element: ElementHandle) -> int:- js_script = "([element]) => getElementDomDepth(element)"+ js_script = with_dom_utils("([element]) => getElementDomDepth(element)", ("getElementDomDepth",))return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element])async def remove_all_unique_ids(self) -> None:- js_script = "() => removeAllUniqueIds()"+ js_script = with_dom_utils("() => removeAllUniqueIds()", ("removeAllUniqueIds",))await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def _set_enriched_element_tree_flag(@@ -2314,7 +2398,10 @@ class SkyvernFrame:# unconditional: it is protection against a hostile wrapper injecting the key, not capture# cost.capture_destination_facts = policy_observation_enabled()- js_script = "async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)"+ js_script = with_dom_utils(+ "async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)",+ ("buildTreeFromBody",),+ )# One monotonic budget across the flag write, both build attempts and the re-injection# between them -- as _evaluate_with_navigation_recovery does -- so the retry cannot double@@ -2339,9 +2426,7 @@ class SkyvernFrame:tree = _as_element_tree_pair(await build())if tree is None:- # Callers scraping an iframe swallow a raise here and drop the whole frame from the tree,- # so spend one re-injection before giving up: a silent non-pair is the same lost JS world- # the raised-ReferenceError path already recovers from, minus an error to key on.+ # Re-bootstrap once for a malformed tree result; missing helpers are prepared before build().LOG.warning("Element tree builder returned no tree, re-injecting domUtils.js and retrying",url=redact_url_secrets(self.frame.url),@@ -2370,7 +2455,10 @@ class SkyvernFrame:timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,) -> tuple[list[dict], list[dict]]:await self._set_enriched_element_tree_flag()- js_script = "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)"+ js_script = with_dom_utils(+ "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)",+ ("getIncrementElements",),+ )result = await self.evaluate(frame=self.frame,engine_selection=self.engine_selection,@@ -2400,7 +2488,10 @@ class SkyvernFrame:timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,) -> tuple[list[dict], list[dict]]:await self._set_enriched_element_tree_flag()- js_script = "async ([starter, frame, full_tree]) => await buildElementTree(starter, frame, full_tree)"+ js_script = with_dom_utils(+ "async ([starter, frame, full_tree]) => await buildElementTree(starter, frame, full_tree)",+ ("buildElementTree",),+ )result = await self.evaluate(frame=self.frame,engine_selection=self.engine_selection,@@ -2449,7 +2540,7 @@ class SkyvernFrame:is_finished = await self.evaluate(frame=self.frame,engine_selection=self.engine_selection,- expression="() => isAnimationFinished()",+ expression=with_dom_utils("() => isAnimationFinished()", ("isAnimationFinished",)),timeout_ms=timeout_ms,)if is_finished:diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.pyindex 0a984e9f0..07067bc3e 100644--- a/tests/unit/test_copilot_credential_pause.py+++ b/tests/unit/test_copilot_credential_pause.py@@ -1778,9 +1778,13 @@ async def test_the_card_outlives_the_model_stream_deadline_the_tool_call_ran_undcard cancelled out from under them."""ctx = _tool_ctx(monkeypatch,- _SlowAnswerCache(credential_response_cache_key("org-1", "chat-1", "turn-1"), 1.6),+ _SlowAnswerCache(credential_response_cache_key("org-1", "chat-1", "turn-1"), 4.5),)- monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 1.0)+ # Keep the timing order explicit: loaded-runner overhead < turn budget < human+ # wait < credential-card timeout. The neighboring resume test uses the same+ # margins because a one-second turn budget is too small on a cold CI shard.+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 4.0)+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)stream = _make_stream()fake_result = _fake_result()diff --git a/tests/unit/test_navigation_recovery.py b/tests/unit/test_navigation_recovery.pyindex ab8a97f9d..52594187f 100644--- a/tests/unit/test_navigation_recovery.py+++ b/tests/unit/test_navigation_recovery.py@@ -1,17 +1,19 @@from __future__ import annotationsimport asyncio-from unittest.mock import ANY, AsyncMock+from unittest.mock import ANY, AsyncMock, MagicMockimport pytestfrom playwright._impl._errors import Error as PlaywrightErrorfrom skyvern.exceptions import SkyvernPageAnalysisTimeoutfrom skyvern.webeye.utils.page import (+ _DOM_UTILS_MISSING_RESULT_KEY,JS_FUNCTION_DEFS,SkyvernFrame,_is_navigation_context_lost,_wait_for_navigation_settle,+ with_dom_utils,)@@ -25,7 +27,7 @@ class TestIsNavigationContextLost:)def test_reference_error_not_defined(self) -> None:- assert _is_navigation_context_lost("Page.evaluate: ReferenceError: scrollToXY is not defined") is True+ assert _is_navigation_context_lost("Page.evaluate: ReferenceError: scrollToXY is not defined") is Falsedef test_missing_protocol_context(self) -> None:assert (@@ -73,10 +75,6 @@ def _context_destroyed_error() -> PlaywrightError:return PlaywrightError("Page.evaluate: Execution context was destroyed, most likely because of a navigation.")-def _reference_error() -> PlaywrightError:- return PlaywrightError("Page.evaluate: ReferenceError: scrollToXY is not defined")--class TestEvaluateWithNavigationRecovery:@pytest.mark.asyncioasync def test_evaluate_timeout_raises_skyvern_page_analysis_timeout(self) -> None:@@ -138,19 +136,19 @@ class TestEvaluateWithNavigationRecovery:assert result == 42@pytest.mark.asyncio- async def test_recovers_after_reference_error(self) -> None:+ @pytest.mark.parametrize("error_type", [PlaywrightError, RuntimeError])+ async def test_reference_error_propagates_without_recovery(self, error_type: type[Exception]) -> None:frame = AsyncMock()- frame.evaluate = AsyncMock(- side_effect=[- _reference_error(),- None,- "ok",- ]- )+ source_error = error_type("Page.evaluate: ReferenceError: pageOwnedMissingValue is not defined")+ frame.evaluate = AsyncMock(side_effect=source_error)frame.wait_for_load_state = AsyncMock()- result = await SkyvernFrame.evaluate(frame=frame, expression="() => getScrollXY()", timeout_ms=30000)- assert result == "ok"+ with pytest.raises(error_type) as exc_info:+ await SkyvernFrame.evaluate(frame=frame, expression="() => getScrollXY()", timeout_ms=30000)++ assert exc_info.value is source_error+ frame.evaluate.assert_awaited_once_with(expression="() => getScrollXY()", arg=None)+ frame.wait_for_load_state.assert_not_awaited()@pytest.mark.asyncioasync def test_fails_after_max_attempts_exhausted(self) -> None:@@ -239,24 +237,26 @@ class TestGetElementVisible:frame.evaluate.assert_awaited_once()@pytest.mark.asyncio- async def test_missing_helpers_reinjects_and_reresolves(self) -> None:+ async def test_missing_helpers_propagates_without_recovery(self) -> None:frame = AsyncMock()frame.evaluate = AsyncMock(return_value=None)frame.wait_for_load_state = AsyncMock()locator = AsyncMock()locator.count = AsyncMock(return_value=1)- locator.evaluate = AsyncMock(- side_effect=[- PlaywrightError("Locator.evaluate: ReferenceError: isElementVisible is not defined"),- True,- ]- )+ source_error = PlaywrightError("Locator.evaluate: ReferenceError: isElementVisible is not defined")+ locator.evaluate = AsyncMock(side_effect=source_error)- result = await SkyvernFrame(frame).get_element_visible(locator)+ with pytest.raises(PlaywrightError) as exc_info:+ await SkyvernFrame(frame).get_element_visible(locator)- assert result is True- assert locator.evaluate.await_count == 2- frame.evaluate.assert_awaited_once()+ assert exc_info.value is source_error+ locator.evaluate.assert_awaited_once_with(+ with_dom_utils(+ "(element) => isElementVisible(element) && !isHidden(element)", ("isElementVisible", "isHidden")+ )+ )+ frame.evaluate.assert_not_awaited()+ frame.wait_for_load_state.assert_not_awaited()@pytest.mark.asyncioasync def test_uses_locator_evaluation_instead_of_stale_handle_marshalling(self) -> None:@@ -273,7 +273,11 @@ class TestGetElementVisible:result = await SkyvernFrame(frame).get_element_visible(locator)assert result is True- locator.evaluate.assert_awaited_once_with("(element) => isElementVisible(element) && !isHidden(element)")+ locator.evaluate.assert_awaited_once_with(+ with_dom_utils(+ "(element) => isElementVisible(element) && !isHidden(element)", ("isElementVisible", "isHidden")+ )+ )frame.evaluate.assert_not_awaited()@pytest.mark.asyncio@@ -287,3 +291,76 @@ class TestGetElementVisible:assert result is Falselocator.evaluate.assert_not_awaited()frame.evaluate.assert_not_awaited()+++ "failures",+ [+ [asyncio.TimeoutError()],+ [_context_destroyed_error(), asyncio.TimeoutError()],+ [_context_destroyed_error(), None, asyncio.TimeoutError()],+ ],+)+async def test_large_helper_expression_is_bounded_in_timeout_logs(failures, monkeypatch):+ from skyvern.webeye.utils import page as page_utils++ warning = MagicMock()+ monkeypatch.setattr(page_utils.LOG, "warning", warning)+ frame = AsyncMock()+ frame.evaluate.side_effect = failures+ expression = "x" * 135000+ with pytest.raises(SkyvernPageAnalysisTimeout):+ await SkyvernFrame.evaluate(frame=frame, expression=expression)+ logged = [call.kwargs["expression"] for call in warning.call_args_list if "expression" in call.kwargs]+ assert logged+ assert all(value == expression[:200] for value in logged)+++async def test_builtin_operation_ships_bundle_only_after_guard_reports_missing_helpers() -> None:+ frame = AsyncMock()+ frame.evaluate.side_effect = [+ {_DOM_UTILS_MISSING_RESULT_KEY: True},+ None,+ [11, 22],+ ]++ assert await SkyvernFrame(frame).get_scroll_x_y() == [11, 22]++ calls = frame.evaluate.await_args_list+ guarded_expression = calls[0].kwargs["expression"]+ assert JS_FUNCTION_DEFS not in guarded_expression+ assert calls[1].kwargs == {"expression": JS_FUNCTION_DEFS, "arg": None}+ assert calls[2].kwargs["expression"] == guarded_expression+++async def test_builtin_operation_warm_path_uses_one_small_dispatch() -> None:+ frame = AsyncMock()+ frame.evaluate.return_value = [11, 22]++ assert await SkyvernFrame(frame).get_scroll_x_y() == [11, 22]++ frame.evaluate.assert_awaited_once()+ assert JS_FUNCTION_DEFS not in frame.evaluate.await_args.kwargs["expression"]+++async def test_cold_locator_bootstraps_through_locator_evaluation() -> None:+ frame = AsyncMock()+ locator = AsyncMock()+ locator.count.return_value = 1+ locator.evaluate.side_effect = [+ {_DOM_UTILS_MISSING_RESULT_KEY: True},+ None,+ True,+ ]++ assert await SkyvernFrame(frame).get_element_visible(locator) is True++ assert locator.evaluate.await_count == 3+ assert JS_FUNCTION_DEFS not in locator.evaluate.await_args_list[0].args[0]+ assert JS_FUNCTION_DEFS in locator.evaluate.await_args_list[1].args[0]+ assert locator.evaluate.await_args_list[2].args[0] == locator.evaluate.await_args_list[0].args[0]+ frame.evaluate.assert_not_awaited()diff --git a/tests/unit/test_page_evaluate_engine_classification.py b/tests/unit/test_page_evaluate_engine_classification.pyindex c5330e058..bae97ac89 100644--- a/tests/unit/test_page_evaluate_engine_classification.py+++ b/tests/unit/test_page_evaluate_engine_classification.py@@ -234,6 +234,7 @@ async def test_instance_evaluate_forwards_engine_selection(monkeypatch: pytest.Mevaluate_expression: object,timeout_ms: float,engine_selection: object = None,+ bootstrap_expression: object = None,) -> str:captured["engine_selection"] = engine_selectionreturn "ok"@@ -257,6 +258,7 @@ async def test_instance_evaluate_expression_forwards_engine_selection(monkeypatcevaluate_expression: object,timeout_ms: float,engine_selection: object = None,+ bootstrap_expression: object = None,) -> bool:captured["engine_selection"] = engine_selectionreturn Truediff --git a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py b/tests/unit/webeye/test_skyvern_frame_main_world_routing.pyindex 2f23e3af7..3602630a6 100644--- a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py+++ b/tests/unit/webeye/test_skyvern_frame_main_world_routing.py@@ -303,7 +303,7 @@ class TestNavigationRecoveryRouting:page.context.new_cdp_session = AsyncMock(return_value=cdp_session) # type: ignore[attr-defined]try:- result = await SkyvernFrame.evaluate(page, "() => 1", timeout_ms=1000)+ result = await SkyvernFrame.evaluate(page, "() => 1")finally:clear_main_world_prefix(ctx) # type: ignore[arg-type]@@ -336,7 +336,7 @@ class TestNavigationRecoveryRouting:)try:- result = await SkyvernFrame.evaluate(page, "(el) => el.blur()", element_handle_like, timeout_ms=1000)+ result = await SkyvernFrame.evaluate(page, "(el) => el.blur()", element_handle_like)finally:clear_main_world_prefix(ctx) # type: ignore[arg-type]@@ -361,7 +361,7 @@ class TestNavigationRecoveryRouting:])- result = await SkyvernFrame.evaluate(frame, "() => 1", timeout_ms=1000)+ result = await SkyvernFrame.evaluate(frame, "() => 1")assert result == "frame-final"assert frame.evaluate.await_count == 3