Skip to content

Automate browser based workflows with AI

Avoid replaying ordinary JavaScript ReferenceErrors (SKY-15778) (#8489)

Andrew Neilsoneaf99422762e
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 1f57284f4..27528aeb0 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -45,7 +45,7 @@ from skyvern.webeye.scraper.scraped_page import (
json_to_html,
)
from skyvern.webeye.utils.document import get_main_document_loader_id
-from skyvern.webeye.utils.page import SkyvernFrame, load_js_script
+from skyvern.webeye.utils.page import SkyvernFrame, load_js_script, with_dom_utils
if TYPE_CHECKING:
from skyvern.webeye.browser_engine import BrowserEngineSelection
@@ -989,7 +989,9 @@ class IncrementalScrapePage(ElementTreeBuilder):
return self.element_tree_trimmed
async def start_listen_dom_increment(self, element: ElementHandle | None = None) -> None:
- js_script = "async (element) => await startGlobalIncrementalObserver(element)"
+ js_script = with_dom_utils(
+ "async (element) => await startGlobalIncrementalObserver(element)", ("startGlobalIncrementalObserver",)
+ )
await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script, arg=element)
async def stop_listen_dom_increment(self) -> None:
@@ -997,7 +999,9 @@ class IncrementalScrapePage(ElementTreeBuilder):
js_script = "() => window.globalObserverForDOMIncrement === undefined"
if await SkyvernFrame.evaluate(frame=self.skyvern_frame.get_frame(), expression=js_script):
return
- js_script = "async () => await stopGlobalIncrementalObserver()"
+ js_script = with_dom_utils(
+ "async () => await stopGlobalIncrementalObserver()", ("stopGlobalIncrementalObserver",)
+ )
await SkyvernFrame.evaluate(
frame=self.skyvern_frame.get_frame(),
expression=js_script,
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 10c4b77df..36ac29dbb 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -268,6 +268,33 @@ def load_js_script() -> str:
JS_FUNCTION_DEFS = load_js_script()
+
+class _DomUtilsExpression(str):
+ """A guarded built-in expression whose helpers may be bootstrapped before it runs."""
+
+
+_DOM_UTILS_MISSING_RESULT_KEY = "__skyvern_dom_utils_missing__"
+
+
+def _is_dom_utils_missing_result(result: Any) -> bool:
+ return result == {_DOM_UTILS_MISSING_RESULT_KEY: True}
+
+
+def with_dom_utils(expression: str, required_helpers: tuple[str, ...]) -> str:
+ """Guard a built-in call so missing helpers are reported before the operation runs."""
+ textual_enabled = SettingsManager.get_settings().ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE
+ context = skyvern_context.current()
+ enriched_enabled = bool(context and context.enriched_tree_enabled())
+ return _DomUtilsExpression(f"""(arg) => {{
+ if (!{json.dumps(required_helpers)}.every((name) => typeof globalThis[name] === "function")) {{
+ return {{{json.dumps(_DOM_UTILS_MISSING_RESULT_KEY)}: true}};
+ }}
+ window.GlobalEnableAllTextualElements = {json.dumps(textual_enabled)};
+ window.GlobalEnableEnrichedElementTree = {json.dumps(enriched_enabled)};
+ return ({expression})(arg);
+ }}""")
+
+
_NAVIGATION_RECOVERY_MAX_ATTEMPTS = 4
_NAVIGATION_SETTLE_TIMEOUT_MS = 3000
@@ -302,11 +329,7 @@ def _extract_playwright_screenshot_stage(exc: BaseException) -> str:
def _is_navigation_context_lost(error_msg: str) -> bool:
- if "Execution context was destroyed" in error_msg:
- return True
- if "Cannot find context with specified id" in error_msg:
- return True
- return "ReferenceError" in error_msg and "is not defined" in error_msg
+ return "Execution context was destroyed" in error_msg or "Cannot find context with specified id" in error_msg
# Order-locked labels for the selectors in LOADING_INDICATOR_PROBE_JS. The JS ships back a fixed
@@ -608,6 +631,22 @@ async def _dispatch_evaluate(frame: Page | Frame, expression: str, arg: Any | No
return await evaluate_in_main_world(frame, expression, arg)
+async def _dispatch_dom_utils_bootstrap(
+ frame: Page | Frame, operation_arg: Any | None, *, force_cdp: bool = False
+) -> Any:
+ """Install helpers through the same world-selection route as the guarded operation."""
+ if force_cdp:
+ return await _dispatch_evaluate(frame, JS_FUNCTION_DEFS, None, force_cdp=True)
+ if not is_page_like(frame):
+ return await frame.evaluate(expression=JS_FUNCTION_DEFS, arg=None)
+ context = frame.context
+ if context is None or get_main_world_prefix(context) is None:
+ return await frame.evaluate(expression=JS_FUNCTION_DEFS, arg=None)
+ if operation_arg is not None and not _is_json_inlinable(operation_arg):
+ return await frame.evaluate(expression=JS_FUNCTION_DEFS, arg=None)
+ return await evaluate_in_main_world(frame, JS_FUNCTION_DEFS)
+
+
async def _wait_for_navigation_settle(
frame: Page | Frame,
timeout_ms: float,
@@ -1501,12 +1540,16 @@ class SkyvernFrame:
async def evaluate_expression() -> Any:
return await _dispatch_evaluate(frame, expression, arg, force_cdp=force_cdp)
+ async def bootstrap_expression() -> Any:
+ return await _dispatch_dom_utils_bootstrap(frame, arg, force_cdp=force_cdp)
+
return await SkyvernFrame._evaluate_expression(
frame=frame,
expression=expression,
evaluate_expression=evaluate_expression,
timeout_ms=timeout_ms,
engine_selection=engine_selection,
+ bootstrap_expression=bootstrap_expression,
**({"deadline": deadline} if deadline is not None else {}),
)
@@ -1518,7 +1561,23 @@ class SkyvernFrame:
timeout_ms: float,
engine_selection: BrowserEngineSelection | None = None,
deadline: float | None = None,
+ bootstrap_expression: Callable[[], Awaitable[Any]] | None = None,
) -> Any:
+ if isinstance(expression, _DomUtilsExpression):
+ if bootstrap_expression is None:
+ raise RuntimeError("domUtils expression requires a matching-world bootstrap")
+ guarded_evaluate = evaluate_expression
+
+ async def evaluate_expression() -> Any:
+ for _ in range(_NAVIGATION_RECOVERY_MAX_ATTEMPTS):
+ result = await guarded_evaluate()
+ if not _is_dom_utils_missing_result(result):
+ return result
+ # The guarded expression returned before invoking the built-in operation, so
+ # installing helpers and trying it again cannot repeat caller-visible effects.
+ await bootstrap_expression()
+ raise RuntimeError("domUtils helpers remained unavailable after bootstrap")
+
loop = asyncio.get_running_loop()
deadline = deadline if deadline is not None else loop.time() + timeout_ms / 1000
try:
@@ -1528,7 +1587,7 @@ class SkyvernFrame:
skyvern_context.record_browser_timeout(BrowserOperation.EVALUATE)
# Re-raised and handled by the caller (scrape retries / failure classification),
# so this is not the failure boundary; log without a traceback at warning.
- LOG.warning("Skyvern timed out trying to analyze the page", expression=expression)
+ LOG.warning("Skyvern timed out trying to analyze the page", expression=expression[:200])
raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error
except RuntimeError as e:
# `evaluate_in_main_world` raises RuntimeError on Runtime.evaluate
@@ -1594,7 +1653,7 @@ class SkyvernFrame:
if _remaining_seconds() <= 0:
LOG.warning(
"Skyvern timed out trying to analyze the page after navigation recovery",
- expression=expression,
+ expression=expression[:200],
)
raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page")
@@ -1615,7 +1674,7 @@ class SkyvernFrame:
if inject_budget <= 0:
LOG.warning(
"Skyvern timed out trying to analyze the page after navigation recovery",
- expression=expression,
+ expression=expression[:200],
)
raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page")
try:
@@ -1626,7 +1685,7 @@ class SkyvernFrame:
except asyncio.TimeoutError as error:
LOG.warning(
"Skyvern timed out trying to analyze the page during domUtils.js re-injection",
- expression=expression,
+ expression=expression[:200],
exc_info=True,
)
raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error
@@ -1649,7 +1708,7 @@ class SkyvernFrame:
if retry_budget <= 0:
LOG.warning(
"Skyvern timed out trying to analyze the page after navigation recovery",
- expression=expression,
+ expression=expression[:200],
)
raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page")
try:
@@ -1660,7 +1719,9 @@ class SkyvernFrame:
return result
except asyncio.TimeoutError as error:
LOG.warning(
- "Skyvern timed out on retry after JS context re-injection", expression=expression, exc_info=True
+ "Skyvern timed out on retry after JS context re-injection",
+ expression=expression[:200],
+ exc_info=True,
)
raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error
except Exception as retry_err:
@@ -2032,7 +2093,7 @@ class SkyvernFrame:
return await self.frame.content()
async def get_scroll_x_y(self) -> tuple[int, int]:
- js_script = "() => getScrollXY()"
+ js_script = with_dom_utils("() => getScrollXY()", ("getScrollXY",))
return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def get_open_aria_popup_trigger(self) -> dict | None:
@@ -2043,7 +2104,10 @@ class SkyvernFrame:
semantics live in getOpenAriaPopupTrigger in domUtils.js.
"""
try:
- result = await self.evaluate(frame=self.frame, expression="() => getOpenAriaPopupTrigger()")
+ result = await self.evaluate(
+ frame=self.frame,
+ expression=with_dom_utils("() => getOpenAriaPopupTrigger()", ("getOpenAriaPopupTrigger",)),
+ )
except Exception:
LOG.warning(
"Failed to detect open ARIA popup trigger; using default scrolling behavior",
@@ -2053,11 +2117,11 @@ class SkyvernFrame:
return result if isinstance(result, dict) else None
async def get_scroll_width_and_height(self) -> tuple[int, int]:
- js_script = "() => getScrollWidthAndHeight()"
+ js_script = with_dom_utils("() => getScrollWidthAndHeight()", ("getScrollWidthAndHeight",))
return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def scroll_to_x_y(self, x: int, y: int) -> None:
- js_script = "([x, y]) => scrollToXY(x, y)"
+ js_script = with_dom_utils("([x, y]) => scrollToXY(x, y)", ("scrollToXY",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[x, y]
)
@@ -2077,19 +2141,24 @@ class SkyvernFrame:
)
async def scroll_to_element_bottom(self, element: ElementHandle, page_by_page: bool = False) -> None:
- js_script = "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)"
+ js_script = with_dom_utils(
+ "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)", ("scrollToElementBottom",)
+ )
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element, page_by_page]
)
async def scroll_to_element_top(self, element: ElementHandle) -> None:
- js_script = "(element) => scrollToElementTop(element)"
+ js_script = with_dom_utils("(element) => scrollToElementTop(element)", ("scrollToElementTop",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
)
async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> dict:
- js_script = "async ([frame, element, interactable]) => await buildElementObject(frame, element, interactable)"
+ js_script = with_dom_utils(
+ "async ([frame, element, interactable]) => await buildElementObject(frame, element, interactable)",
+ ("buildElementObject",),
+ )
parsed = await self.evaluate(
frame=self.frame,
engine_selection=self.engine_selection,
@@ -2100,35 +2169,41 @@ class SkyvernFrame:
return parsed
async def get_element_scrollable(self, element: ElementHandle) -> bool:
- js_script = "(element) => isScrollable(element)"
+ js_script = with_dom_utils("(element) => isScrollable(element)", ("isScrollable",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
)
async def get_element_visible(self, locator: Locator) -> bool:
- js_script = "(element) => isElementVisible(element) && !isHidden(element)"
+ js_script = with_dom_utils(
+ "(element) => isElementVisible(element) && !isHidden(element)", ("isElementVisible", "isHidden")
+ )
async def evaluate_expression() -> bool:
if await locator.count() == 0:
return False
return await locator.evaluate(js_script)
+ async def bootstrap_expression() -> Any:
+ return await locator.evaluate(f"(element) => {{ {JS_FUNCTION_DEFS} }}")
+
return await self._evaluate_expression(
frame=self.frame,
engine_selection=self.engine_selection,
expression=js_script,
evaluate_expression=evaluate_expression,
+ bootstrap_expression=bootstrap_expression,
timeout_ms=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
)
async def get_disabled_from_style(self, element: ElementHandle) -> bool:
- js_script = "(element) => checkDisabledFromStyle(element)"
+ js_script = with_dom_utils("(element) => checkDisabledFromStyle(element)", ("checkDisabledFromStyle",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
)
async def get_blocking_element_id(self, element: ElementHandle) -> tuple[str, bool]:
- js_script = "(element) => getBlockElementUniqueID(element)"
+ js_script = with_dom_utils("(element) => getBlockElementUniqueID(element)", ("getBlockElementUniqueID",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
)
@@ -2140,7 +2215,10 @@ class SkyvernFrame:
:param page: Page instance to take the screenshot from.
:return: Screenshot of the page.
"""
- js_script = "async ([draw_boxes, frame, frame_index]) => await safeScrollToTop(draw_boxes, frame, frame_index)"
+ js_script = with_dom_utils(
+ "async ([draw_boxes, frame, frame_index]) => await safeScrollToTop(draw_boxes, frame, frame_index)",
+ ("safeScrollToTop",),
+ )
scroll_y_px = await self.evaluate(
frame=self.frame,
engine_selection=self.engine_selection,
@@ -2165,7 +2243,10 @@ class SkyvernFrame:
:param page: Page instance to take the screenshot from.
:return: Screenshot of the page.
"""
- js_script = "async ([draw_boxes, frame, frame_index, need_overlap]) => await scrollToNextPage(draw_boxes, frame, frame_index, need_overlap)"
+ js_script = with_dom_utils(
+ "async ([draw_boxes, frame, frame_index, need_overlap]) => await scrollToNextPage(draw_boxes, frame, frame_index, need_overlap)",
+ ("scrollToNextPage",),
+ )
scroll_y_px = await self.evaluate(
frame=self.frame,
engine_selection=self.engine_selection,
@@ -2186,7 +2267,7 @@ class SkyvernFrame:
Remove the bounding boxes from the page.
:param page: Page instance to remove the bounding boxes from.
"""
- js_script = "() => removeBoundingBoxes()"
+ js_script = with_dom_utils("() => removeBoundingBoxes()", ("removeBoundingBoxes",))
await self.evaluate(
frame=self.frame,
engine_selection=self.engine_selection,
@@ -2195,7 +2276,10 @@ class SkyvernFrame:
)
async def build_elements_and_draw_bounding_boxes(self, frame: str, frame_index: int) -> None:
- js_script = "async ([frame, frame_index]) => await buildElementsAndDrawBoundingBoxes(frame, frame_index)"
+ js_script = with_dom_utils(
+ "async ([frame, frame_index]) => await buildElementsAndDrawBoundingBoxes(frame, frame_index)",
+ ("buildElementsAndDrawBoundingBoxes",),
+ )
await self.evaluate(
frame=self.frame,
engine_selection=self.engine_selection,
@@ -2205,23 +2289,23 @@ class SkyvernFrame:
)
async def is_window_scrollable(self) -> bool:
- js_script = "() => isWindowScrollable()"
+ js_script = with_dom_utils("() => isWindowScrollable()", ("isWindowScrollable",))
return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def is_parent(self, parent: ElementHandle, child: ElementHandle) -> bool:
- js_script = "([parent, child]) => isParent(parent, child)"
+ js_script = with_dom_utils("([parent, child]) => isParent(parent, child)", ("isParent",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[parent, child]
)
async def is_sibling(self, el1: ElementHandle, el2: ElementHandle) -> bool:
- js_script = "([el1, el2]) => isSibling(el1, el2)"
+ js_script = with_dom_utils("([el1, el2]) => isSibling(el1, el2)", ("isSibling",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[el1, el2]
)
async def has_ASP_client_control(self) -> bool:
- js_script = "() => hasASPClientControl()"
+ js_script = with_dom_utils("() => hasASPClientControl()", ("hasASPClientControl",))
return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def click_element_in_javascript(self, element: ElementHandle) -> None:
@@ -2267,19 +2351,19 @@ class SkyvernFrame:
)
async def get_select_options(self, element: ElementHandle) -> tuple[list, str]:
- js_script = "([element]) => getSelectOptions(element)"
+ js_script = with_dom_utils("([element]) => getSelectOptions(element)", ("getSelectOptions",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element]
)
async def get_element_dom_depth(self, element: ElementHandle) -> int:
- js_script = "([element]) => getElementDomDepth(element)"
+ js_script = with_dom_utils("([element]) => getElementDomDepth(element)", ("getElementDomDepth",))
return await self.evaluate(
frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element]
)
async def remove_all_unique_ids(self) -> None:
- js_script = "() => removeAllUniqueIds()"
+ js_script = with_dom_utils("() => removeAllUniqueIds()", ("removeAllUniqueIds",))
await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def _set_enriched_element_tree_flag(
@@ -2314,7 +2398,10 @@ class SkyvernFrame:
# unconditional: it is protection against a hostile wrapper injecting the key, not capture
# cost.
capture_destination_facts = policy_observation_enabled()
- js_script = "async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)"
+ js_script = with_dom_utils(
+ "async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)",
+ ("buildTreeFromBody",),
+ )
# One monotonic budget across the flag write, both build attempts and the re-injection
# between them -- as _evaluate_with_navigation_recovery does -- so the retry cannot double
@@ -2339,9 +2426,7 @@ class SkyvernFrame:
tree = _as_element_tree_pair(await build())
if tree is None:
- # Callers scraping an iframe swallow a raise here and drop the whole frame from the tree,
- # so spend one re-injection before giving up: a silent non-pair is the same lost JS world
- # the raised-ReferenceError path already recovers from, minus an error to key on.
+ # Re-bootstrap once for a malformed tree result; missing helpers are prepared before build().
LOG.warning(
"Element tree builder returned no tree, re-injecting domUtils.js and retrying",
url=redact_url_secrets(self.frame.url),
@@ -2370,7 +2455,10 @@ class SkyvernFrame:
timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
) -> tuple[list[dict], list[dict]]:
await self._set_enriched_element_tree_flag()
- js_script = "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)"
+ js_script = with_dom_utils(
+ "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)",
+ ("getIncrementElements",),
+ )
result = await self.evaluate(
frame=self.frame,
engine_selection=self.engine_selection,
@@ -2400,7 +2488,10 @@ class SkyvernFrame:
timeout_ms: float = SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
) -> tuple[list[dict], list[dict]]:
await self._set_enriched_element_tree_flag()
- js_script = "async ([starter, frame, full_tree]) => await buildElementTree(starter, frame, full_tree)"
+ js_script = with_dom_utils(
+ "async ([starter, frame, full_tree]) => await buildElementTree(starter, frame, full_tree)",
+ ("buildElementTree",),
+ )
result = await self.evaluate(
frame=self.frame,
engine_selection=self.engine_selection,
@@ -2449,7 +2540,7 @@ class SkyvernFrame:
is_finished = await self.evaluate(
frame=self.frame,
engine_selection=self.engine_selection,
- expression="() => isAnimationFinished()",
+ expression=with_dom_utils("() => isAnimationFinished()", ("isAnimationFinished",)),
timeout_ms=timeout_ms,
)
if is_finished:
diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.py
index 0a984e9f0..07067bc3e 100644
--- a/tests/unit/test_copilot_credential_pause.py
+++ b/tests/unit/test_copilot_credential_pause.py
@@ -1778,9 +1778,13 @@ async def test_the_card_outlives_the_model_stream_deadline_the_tool_call_ran_und
card cancelled out from under them."""
ctx = _tool_ctx(
monkeypatch,
- _SlowAnswerCache(credential_response_cache_key("org-1", "chat-1", "turn-1"), 1.6),
+ _SlowAnswerCache(credential_response_cache_key("org-1", "chat-1", "turn-1"), 4.5),
)
- monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 1.0)
+ # Keep the timing order explicit: loaded-runner overhead < turn budget < human
+ # wait < credential-card timeout. The neighboring resume test uses the same
+ # margins because a one-second turn budget is too small on a cold CI shard.
+ monkeypatch.setattr("skyvern.forge.sdk.copilot.enforcement.TOTAL_TIMEOUT_SECONDS", 4.0)
+ ctx.copilot_config = CopilotConfig(credential_pause_enabled=True, credential_pause_timeout_seconds=30)
stream = _make_stream()
fake_result = _fake_result()
diff --git a/tests/unit/test_navigation_recovery.py b/tests/unit/test_navigation_recovery.py
index ab8a97f9d..52594187f 100644
--- a/tests/unit/test_navigation_recovery.py
+++ b/tests/unit/test_navigation_recovery.py
@@ -1,17 +1,19 @@
from __future__ import annotations
import asyncio
-from unittest.mock import ANY, AsyncMock
+from unittest.mock import ANY, AsyncMock, MagicMock
import pytest
from playwright._impl._errors import Error as PlaywrightError
from skyvern.exceptions import SkyvernPageAnalysisTimeout
from skyvern.webeye.utils.page import (
+ _DOM_UTILS_MISSING_RESULT_KEY,
JS_FUNCTION_DEFS,
SkyvernFrame,
_is_navigation_context_lost,
_wait_for_navigation_settle,
+ with_dom_utils,
)
@@ -25,7 +27,7 @@ class TestIsNavigationContextLost:
)
def test_reference_error_not_defined(self) -> None:
- assert _is_navigation_context_lost("Page.evaluate: ReferenceError: scrollToXY is not defined") is True
+ assert _is_navigation_context_lost("Page.evaluate: ReferenceError: scrollToXY is not defined") is False
def test_missing_protocol_context(self) -> None:
assert (
@@ -73,10 +75,6 @@ def _context_destroyed_error() -> PlaywrightError:
return PlaywrightError("Page.evaluate: Execution context was destroyed, most likely because of a navigation.")
-def _reference_error() -> PlaywrightError:
- return PlaywrightError("Page.evaluate: ReferenceError: scrollToXY is not defined")
-
-
class TestEvaluateWithNavigationRecovery:
@pytest.mark.asyncio
async def test_evaluate_timeout_raises_skyvern_page_analysis_timeout(self) -> None:
@@ -138,19 +136,19 @@ class TestEvaluateWithNavigationRecovery:
assert result == 42
@pytest.mark.asyncio
- async def test_recovers_after_reference_error(self) -> None:
+ @pytest.mark.parametrize("error_type", [PlaywrightError, RuntimeError])
+ async def test_reference_error_propagates_without_recovery(self, error_type: type[Exception]) -> None:
frame = AsyncMock()
- frame.evaluate = AsyncMock(
- side_effect=[
- _reference_error(),
- None,
- "ok",
- ]
- )
+ source_error = error_type("Page.evaluate: ReferenceError: pageOwnedMissingValue is not defined")
+ frame.evaluate = AsyncMock(side_effect=source_error)
frame.wait_for_load_state = AsyncMock()
- result = await SkyvernFrame.evaluate(frame=frame, expression="() => getScrollXY()", timeout_ms=30000)
- assert result == "ok"
+ with pytest.raises(error_type) as exc_info:
+ await SkyvernFrame.evaluate(frame=frame, expression="() => getScrollXY()", timeout_ms=30000)
+
+ assert exc_info.value is source_error
+ frame.evaluate.assert_awaited_once_with(expression="() => getScrollXY()", arg=None)
+ frame.wait_for_load_state.assert_not_awaited()
@pytest.mark.asyncio
async def test_fails_after_max_attempts_exhausted(self) -> None:
@@ -239,24 +237,26 @@ class TestGetElementVisible:
frame.evaluate.assert_awaited_once()
@pytest.mark.asyncio
- async def test_missing_helpers_reinjects_and_reresolves(self) -> None:
+ async def test_missing_helpers_propagates_without_recovery(self) -> None:
frame = AsyncMock()
frame.evaluate = AsyncMock(return_value=None)
frame.wait_for_load_state = AsyncMock()
locator = AsyncMock()
locator.count = AsyncMock(return_value=1)
- locator.evaluate = AsyncMock(
- side_effect=[
- PlaywrightError("Locator.evaluate: ReferenceError: isElementVisible is not defined"),
- True,
- ]
- )
+ source_error = PlaywrightError("Locator.evaluate: ReferenceError: isElementVisible is not defined")
+ locator.evaluate = AsyncMock(side_effect=source_error)
- result = await SkyvernFrame(frame).get_element_visible(locator)
+ with pytest.raises(PlaywrightError) as exc_info:
+ await SkyvernFrame(frame).get_element_visible(locator)
- assert result is True
- assert locator.evaluate.await_count == 2
- frame.evaluate.assert_awaited_once()
+ assert exc_info.value is source_error
+ locator.evaluate.assert_awaited_once_with(
+ with_dom_utils(
+ "(element) => isElementVisible(element) && !isHidden(element)", ("isElementVisible", "isHidden")
+ )
+ )
+ frame.evaluate.assert_not_awaited()
+ frame.wait_for_load_state.assert_not_awaited()
@pytest.mark.asyncio
async def test_uses_locator_evaluation_instead_of_stale_handle_marshalling(self) -> None:
@@ -273,7 +273,11 @@ class TestGetElementVisible:
result = await SkyvernFrame(frame).get_element_visible(locator)
assert result is True
- locator.evaluate.assert_awaited_once_with("(element) => isElementVisible(element) && !isHidden(element)")
+ locator.evaluate.assert_awaited_once_with(
+ with_dom_utils(
+ "(element) => isElementVisible(element) && !isHidden(element)", ("isElementVisible", "isHidden")
+ )
+ )
frame.evaluate.assert_not_awaited()
@pytest.mark.asyncio
@@ -287,3 +291,76 @@ class TestGetElementVisible:
assert result is False
locator.evaluate.assert_not_awaited()
frame.evaluate.assert_not_awaited()
+
+
+ "failures",
+ [
+ [asyncio.TimeoutError()],
+ [_context_destroyed_error(), asyncio.TimeoutError()],
+ [_context_destroyed_error(), None, asyncio.TimeoutError()],
+ ],
+)
+async def test_large_helper_expression_is_bounded_in_timeout_logs(failures, monkeypatch):
+ from skyvern.webeye.utils import page as page_utils
+
+ warning = MagicMock()
+ monkeypatch.setattr(page_utils.LOG, "warning", warning)
+ frame = AsyncMock()
+ frame.evaluate.side_effect = failures
+ expression = "x" * 135000
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ await SkyvernFrame.evaluate(frame=frame, expression=expression)
+ logged = [call.kwargs["expression"] for call in warning.call_args_list if "expression" in call.kwargs]
+ assert logged
+ assert all(value == expression[:200] for value in logged)
+
+
+async def test_builtin_operation_ships_bundle_only_after_guard_reports_missing_helpers() -> None:
+ frame = AsyncMock()
+ frame.evaluate.side_effect = [
+ {_DOM_UTILS_MISSING_RESULT_KEY: True},
+ None,
+ [11, 22],
+ ]
+
+ assert await SkyvernFrame(frame).get_scroll_x_y() == [11, 22]
+
+ calls = frame.evaluate.await_args_list
+ guarded_expression = calls[0].kwargs["expression"]
+ assert JS_FUNCTION_DEFS not in guarded_expression
+ assert calls[1].kwargs == {"expression": JS_FUNCTION_DEFS, "arg": None}
+ assert calls[2].kwargs["expression"] == guarded_expression
+
+
+async def test_builtin_operation_warm_path_uses_one_small_dispatch() -> None:
+ frame = AsyncMock()
+ frame.evaluate.return_value = [11, 22]
+
+ assert await SkyvernFrame(frame).get_scroll_x_y() == [11, 22]
+
+ frame.evaluate.assert_awaited_once()
+ assert JS_FUNCTION_DEFS not in frame.evaluate.await_args.kwargs["expression"]
+
+
+async def test_cold_locator_bootstraps_through_locator_evaluation() -> None:
+ frame = AsyncMock()
+ locator = AsyncMock()
+ locator.count.return_value = 1
+ locator.evaluate.side_effect = [
+ {_DOM_UTILS_MISSING_RESULT_KEY: True},
+ None,
+ True,
+ ]
+
+ assert await SkyvernFrame(frame).get_element_visible(locator) is True
+
+ assert locator.evaluate.await_count == 3
+ assert JS_FUNCTION_DEFS not in locator.evaluate.await_args_list[0].args[0]
+ assert JS_FUNCTION_DEFS in locator.evaluate.await_args_list[1].args[0]
+ assert locator.evaluate.await_args_list[2].args[0] == locator.evaluate.await_args_list[0].args[0]
+ frame.evaluate.assert_not_awaited()
diff --git a/tests/unit/test_page_evaluate_engine_classification.py b/tests/unit/test_page_evaluate_engine_classification.py
index c5330e058..bae97ac89 100644
--- a/tests/unit/test_page_evaluate_engine_classification.py
+++ b/tests/unit/test_page_evaluate_engine_classification.py
@@ -234,6 +234,7 @@ async def test_instance_evaluate_forwards_engine_selection(monkeypatch: pytest.M
evaluate_expression: object,
timeout_ms: float,
engine_selection: object = None,
+ bootstrap_expression: object = None,
) -> str:
captured["engine_selection"] = engine_selection
return "ok"
@@ -257,6 +258,7 @@ async def test_instance_evaluate_expression_forwards_engine_selection(monkeypatc
evaluate_expression: object,
timeout_ms: float,
engine_selection: object = None,
+ bootstrap_expression: object = None,
) -> bool:
captured["engine_selection"] = engine_selection
return True
diff --git a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py b/tests/unit/webeye/test_skyvern_frame_main_world_routing.py
index 2f23e3af7..3602630a6 100644
--- a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py
+++ b/tests/unit/webeye/test_skyvern_frame_main_world_routing.py
@@ -303,7 +303,7 @@ class TestNavigationRecoveryRouting:
page.context.new_cdp_session = AsyncMock(return_value=cdp_session) # type: ignore[attr-defined]
try:
- result = await SkyvernFrame.evaluate(page, "() => 1", timeout_ms=1000)
+ result = await SkyvernFrame.evaluate(page, "() => 1")
finally:
clear_main_world_prefix(ctx) # type: ignore[arg-type]
@@ -336,7 +336,7 @@ class TestNavigationRecoveryRouting:
)
try:
- result = await SkyvernFrame.evaluate(page, "(el) => el.blur()", element_handle_like, timeout_ms=1000)
+ result = await SkyvernFrame.evaluate(page, "(el) => el.blur()", element_handle_like)
finally:
clear_main_world_prefix(ctx) # type: ignore[arg-type]
@@ -361,7 +361,7 @@ class TestNavigationRecoveryRouting:
]
)
- result = await SkyvernFrame.evaluate(frame, "() => 1", timeout_ms=1000)
+ result = await SkyvernFrame.evaluate(frame, "() => 1")
assert result == "frame-final"
assert frame.evaluate.await_count == 3