Skip to content

Automate browser based workflows with AI

Route page.py evaluate/context-loss recovery through the selected browser engine (RUS-5) [R4a] (#7811)

LawyZhenge5a37c922ea0
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index dced0c5c7..ee8738e6f 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -24,6 +24,7 @@ from skyvern.forge.sdk.browser_action_preflight import policy_observation_enable
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import apply_context_attrs, traced
+from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.browser_object_predicates import is_page_like
from skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefix
@@ -139,6 +140,13 @@ _NAVIGATION_RECOVERY_MAX_ATTEMPTS = 4
_NAVIGATION_SETTLE_TIMEOUT_MS = 3000
+def _is_engine_error(exc: BaseException, engine_selection: BrowserEngineSelection | None) -> bool:
+ """Whether ``exc`` belongs to THIS run's selected browser-engine error family. Falls back to the
+ stock-Playwright identity when no engine is pinned, so an unmigrated caller keeps exact stock
+ behavior; a foreign engine's error (or an unrelated exception) is rejected and left to propagate."""
+ return engine_selection.is_engine_error(exc) if engine_selection is not None else isinstance(exc, PlaywrightError)
+
+
def _is_navigation_context_lost(error_msg: str) -> bool:
if "Execution context was destroyed" in error_msg:
return True
@@ -174,12 +182,18 @@ async def _dispatch_evaluate(frame: Page | Frame, expression: str, arg: Any | No
return await evaluate_in_main_world(frame, expression, arg)
-async def _wait_for_navigation_settle(frame: Page | Frame, timeout_ms: float) -> None:
+async def _wait_for_navigation_settle(
+ frame: Page | Frame,
+ timeout_ms: float,
+ engine_selection: BrowserEngineSelection | None = None,
+) -> None:
if timeout_ms <= 0:
return
try:
await frame.wait_for_load_state("networkidle", timeout=timeout_ms)
- except PlaywrightError:
+ except Exception as e:
+ if not _is_engine_error(e, engine_selection):
+ raise
return
@@ -604,12 +618,15 @@ def pop_destination_facts(nodes: object) -> dict[str, dict]:
class SkyvernFrame:
+ engine_selection: BrowserEngineSelection | None = None
+
@staticmethod
async def evaluate(
frame: Page | Frame,
expression: str,
arg: Any | None = None,
timeout_ms: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
+ engine_selection: BrowserEngineSelection | None = None,
) -> Any:
async def evaluate_expression() -> Any:
return await _dispatch_evaluate(frame, expression, arg)
@@ -619,6 +636,7 @@ class SkyvernFrame:
expression=expression,
evaluate_expression=evaluate_expression,
timeout_ms=timeout_ms,
+ engine_selection=engine_selection,
)
@staticmethod
@@ -627,11 +645,19 @@ class SkyvernFrame:
expression: str,
evaluate_expression: Callable[[], Awaitable[Any]],
timeout_ms: float,
+ engine_selection: BrowserEngineSelection | None = None,
) -> Any:
try:
async with asyncio.timeout(timeout_ms / 1000):
return await evaluate_expression()
- except PlaywrightError as e:
+ except asyncio.TimeoutError as error:
+ # Re-raised and handled by the caller (scrape retries / failure classification),
+ # so this is not the failure boundary; log without a traceback at warning.
+ LOG.warning("Skyvern timed out trying to analyze the page", expression=expression)
+ raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error
+ except RuntimeError as e:
+ # `evaluate_in_main_world` raises RuntimeError on Runtime.evaluate
+ # exception payloads; only navigation-context-lost text recovers here.
error_msg = str(e)
if not _is_navigation_context_lost(error_msg):
raise
@@ -641,10 +667,14 @@ class SkyvernFrame:
evaluate_expression=evaluate_expression,
timeout_ms=timeout_ms,
initial_error=error_msg,
+ engine_selection=engine_selection,
)
- except RuntimeError as e:
- # `evaluate_in_main_world` raises RuntimeError on Runtime.evaluate
- # exception payloads; only navigation-context-lost text recovers here.
+ except Exception as e:
+ # A driver-native error from THIS run's engine (stock Playwright when no engine is
+ # pinned). A foreign engine's error, or an unrelated exception, is not ours: re-raise it
+ # unchanged. Cancellation (BaseException, not Exception) is never caught here.
+ if not _is_engine_error(e, engine_selection):
+ raise
error_msg = str(e)
if not _is_navigation_context_lost(error_msg):
raise
@@ -654,12 +684,8 @@ class SkyvernFrame:
evaluate_expression=evaluate_expression,
timeout_ms=timeout_ms,
initial_error=error_msg,
+ engine_selection=engine_selection,
)
- except asyncio.TimeoutError as error:
- # Re-raised and handled by the caller (scrape retries / failure classification),
- # so this is not the failure boundary; log without a traceback at warning.
- LOG.warning("Skyvern timed out trying to analyze the page", expression=expression)
- raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error
@staticmethod
async def _evaluate_with_navigation_recovery(
@@ -668,6 +694,7 @@ class SkyvernFrame:
evaluate_expression: Callable[[], Awaitable[Any]],
timeout_ms: float,
initial_error: str,
+ engine_selection: BrowserEngineSelection | None = None,
) -> Any:
# Multi-hop SSO/OIDC flows (especially response_mode=form_post) can destroy
# the JS execution context several times in a row as the page auto-submits
@@ -697,7 +724,7 @@ class SkyvernFrame:
error=last_error_msg[:200],
)
settle_ms = min(_NAVIGATION_SETTLE_TIMEOUT_MS, _remaining_seconds() * 1000)
- await _wait_for_navigation_settle(frame, timeout_ms=settle_ms)
+ await _wait_for_navigation_settle(frame, timeout_ms=settle_ms, engine_selection=engine_selection)
# The bootstrap call already IS the domUtils.js injection, so the retry below
# re-injects it anyway; a separate injection pass would spend a second attempt
@@ -721,7 +748,12 @@ class SkyvernFrame:
expression=expression,
)
raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error
- except (PlaywrightError, RuntimeError) as inject_err:
+ except Exception as inject_err:
+ # RuntimeError (main-world Runtime.evaluate payloads) is engine-agnostic; the
+ # driver-native failure routes through the per-run engine identity. A foreign
+ # error is re-raised; cancellation never reaches here.
+ if not (isinstance(inject_err, RuntimeError) or _is_engine_error(inject_err, engine_selection)):
+ raise
last_error_msg = str(inject_err)
if attempt == _NAVIGATION_RECOVERY_MAX_ATTEMPTS or not _is_navigation_context_lost(last_error_msg):
LOG.warning(
@@ -744,7 +776,9 @@ class SkyvernFrame:
except asyncio.TimeoutError as error:
LOG.exception("Skyvern timed out on retry after JS context re-injection", expression=expression)
raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error
- except (PlaywrightError, RuntimeError) as retry_err:
+ except Exception as retry_err:
+ if not (isinstance(retry_err, RuntimeError) or _is_engine_error(retry_err, engine_selection)):
+ raise
last_error_msg = str(retry_err)
if attempt == _NAVIGATION_RECOVERY_MAX_ATTEMPTS or not _is_navigation_context_lost(last_error_msg):
raise
@@ -1038,17 +1072,25 @@ class SkyvernFrame:
return screenshots
@classmethod
- async def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:
- instance = cls(frame=frame)
- await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS)
+ async def create_instance(
+ cls, frame: Page | Frame, engine_selection: BrowserEngineSelection | None = None
+ ) -> SkyvernFrame:
+ instance = cls(frame=frame, engine_selection=engine_selection)
+ await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS, engine_selection=engine_selection)
if SettingsManager.get_settings().ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:
await instance.evaluate(
- frame=instance.frame, expression="() => window.GlobalEnableAllTextualElements = true"
+ frame=instance.frame,
+ expression="() => window.GlobalEnableAllTextualElements = true",
+ engine_selection=engine_selection,
)
return instance
- def __init__(self, frame: Page | Frame) -> None:
+ def __init__(self, frame: Page | Frame, engine_selection: BrowserEngineSelection | None = None) -> None:
self.frame = frame
+ # The logical run's pinned engine (None when the frame was created outside the per-run engine
+ # seam), forwarded to every evaluate so navigation-context-loss recovery keys on THIS engine's
+ # native error family; None preserves the exact stock-Playwright behavior.
+ self.engine_selection = engine_selection
def get_frame(self) -> Page | Frame:
return self.frame
@@ -1060,15 +1102,17 @@ class SkyvernFrame:
async def get_scroll_x_y(self) -> tuple[int, int]:
js_script = "() => getScrollXY()"
- return await self.evaluate(frame=self.frame, expression=js_script)
+ return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def get_scroll_width_and_height(self) -> tuple[int, int]:
js_script = "() => getScrollWidthAndHeight()"
- return await self.evaluate(frame=self.frame, expression=js_script)
+ return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def scroll_to_x_y(self, x: int, y: int) -> None:
js_script = "([x, y]) => scrollToXY(x, y)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=[x, y])
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[x, y]
+ )
async def safe_scroll_to_x_y(self, x: int, y: int) -> None:
try:
@@ -1080,25 +1124,38 @@ class SkyvernFrame:
"""Scroll all ancestor containers (including nested ones with overflow-y: auto)
so that the element is centered in the viewport."""
js_script = "(element) => element.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'})"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
+ )
async def scroll_to_element_bottom(self, element: ElementHandle, page_by_page: bool = False) -> None:
js_script = "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=[element, page_by_page])
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element, page_by_page]
+ )
async def scroll_to_element_top(self, element: ElementHandle) -> None:
js_script = "(element) => scrollToElementTop(element)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
+ )
async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> dict:
js_script = "async ([frame, element, interactable]) => await buildElementObject(frame, element, interactable)"
- parsed = await self.evaluate(frame=self.frame, expression=js_script, arg=[frame, element, interactable])
+ parsed = await self.evaluate(
+ frame=self.frame,
+ engine_selection=self.engine_selection,
+ expression=js_script,
+ arg=[frame, element, interactable],
+ )
pop_destination_facts([parsed])
return parsed
async def get_element_scrollable(self, element: ElementHandle) -> bool:
js_script = "(element) => isScrollable(element)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
+ )
async def get_element_visible(self, locator: Locator) -> bool:
js_script = "(element) => isElementVisible(element) && !isHidden(element)"
@@ -1110,6 +1167,7 @@ class SkyvernFrame:
return await self._evaluate_expression(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression=js_script,
evaluate_expression=evaluate_expression,
timeout_ms=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
@@ -1117,11 +1175,15 @@ class SkyvernFrame:
async def get_disabled_from_style(self, element: ElementHandle) -> bool:
js_script = "(element) => checkDisabledFromStyle(element)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
+ )
async def get_blocking_element_id(self, element: ElementHandle) -> tuple[str, bool]:
js_script = "(element) => getBlockElementUniqueID(element)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
+ )
async def scroll_to_top(self, draw_boxes: bool, frame: str, frame_index: int) -> float:
"""
@@ -1133,6 +1195,7 @@ class SkyvernFrame:
js_script = "async ([draw_boxes, frame, frame_index]) => await safeScrollToTop(draw_boxes, frame, frame_index)"
scroll_y_px = await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression=js_script,
timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
arg=[draw_boxes, frame, frame_index],
@@ -1157,6 +1220,7 @@ class SkyvernFrame:
js_script = "async ([draw_boxes, frame, frame_index, need_overlap]) => await scrollToNextPage(draw_boxes, frame, frame_index, need_overlap)"
scroll_y_px = await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression=js_script,
timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
arg=[draw_boxes, frame, frame_index, need_overlap],
@@ -1177,6 +1241,7 @@ class SkyvernFrame:
js_script = "() => removeBoundingBoxes()"
await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression=js_script,
timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
)
@@ -1185,6 +1250,7 @@ class SkyvernFrame:
js_script = "async ([frame, frame_index]) => await buildElementsAndDrawBoundingBoxes(frame, frame_index)"
await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression=js_script,
timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
arg=[frame, frame_index],
@@ -1192,23 +1258,29 @@ class SkyvernFrame:
async def is_window_scrollable(self) -> bool:
js_script = "() => isWindowScrollable()"
- return await self.evaluate(frame=self.frame, expression=js_script)
+ return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def is_parent(self, parent: ElementHandle, child: ElementHandle) -> bool:
js_script = "([parent, child]) => isParent(parent, child)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=[parent, child])
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[parent, child]
+ )
async def is_sibling(self, el1: ElementHandle, el2: ElementHandle) -> bool:
js_script = "([el1, el2]) => isSibling(el1, el2)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=[el1, el2])
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[el1, el2]
+ )
async def has_ASP_client_control(self) -> bool:
js_script = "() => hasASPClientControl()"
- return await self.evaluate(frame=self.frame, expression=js_script)
+ return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def click_element_in_javascript(self, element: ElementHandle) -> None:
js_script = "(element) => element.click()"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
+ )
async def read_autocomplete_option_identity(self, element: ElementHandle) -> dict[str, Any] | None:
js_script = r"""
@@ -1235,30 +1307,39 @@ class SkyvernFrame:
return { index: optionNodes.indexOf(node), label };
}
"""
- identity = await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+ identity = await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
+ )
return identity if isinstance(identity, dict) else None
async def remove_target_attr(self, element: ElementHandle) -> None:
js_script = "(element) => element.removeAttribute('target')"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element
+ )
async def get_select_options(self, element: ElementHandle) -> tuple[list, str]:
js_script = "([element]) => getSelectOptions(element)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=[element])
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element]
+ )
async def get_element_dom_depth(self, element: ElementHandle) -> int:
js_script = "([element]) => getElementDomDepth(element)"
- return await self.evaluate(frame=self.frame, expression=js_script, arg=[element])
+ return await self.evaluate(
+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element]
+ )
async def remove_all_unique_ids(self) -> None:
js_script = "() => removeAllUniqueIds()"
- await self.evaluate(frame=self.frame, expression=js_script)
+ await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)
async def _set_enriched_element_tree_flag(self) -> None:
context = skyvern_context.current()
enriched_enabled = bool(context and context.enriched_tree_enabled())
await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression="([enabled]) => { window.GlobalEnableEnrichedElementTree = enabled; }",
arg=[enriched_enabled],
)
@@ -1282,6 +1363,7 @@ class SkyvernFrame:
js_script = "async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)"
elements, element_tree = await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression=js_script,
timeout_ms=timeout_ms,
arg=[frame_name, frame_index, must_included_tags, capture_destination_facts],
@@ -1299,7 +1381,11 @@ class SkyvernFrame:
await self._set_enriched_element_tree_flag()
js_script = "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)"
elements, element_tree = await self.evaluate(
- frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[wait_until_finished]
+ frame=self.frame,
+ engine_selection=self.engine_selection,
+ expression=js_script,
+ timeout_ms=timeout_ms,
+ arg=[wait_until_finished],
)
pop_destination_facts(elements)
pop_destination_facts(element_tree)
@@ -1316,7 +1402,11 @@ class SkyvernFrame:
await self._set_enriched_element_tree_flag()
js_script = "async ([starter, frame, full_tree]) => await buildElementTree(starter, frame, full_tree)"
elements, element_tree = await self.evaluate(
- frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[starter, frame, full_tree]
+ frame=self.frame,
+ engine_selection=self.engine_selection,
+ expression=js_script,
+ timeout_ms=timeout_ms,
+ arg=[starter, frame, full_tree],
)
pop_destination_facts(elements)
pop_destination_facts(element_tree)
@@ -1354,6 +1444,7 @@ class SkyvernFrame:
while True:
is_finished = await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression="() => isAnimationFinished()",
timeout_ms=timeout_ms,
)
@@ -1494,6 +1585,7 @@ class SkyvernFrame:
while True:
has_loading_indicator = await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression=loading_indicator_js,
timeout_ms=timeout_ms,
)
@@ -1558,6 +1650,7 @@ class SkyvernFrame:
await self.evaluate(
frame=self.frame,
+ engine_selection=self.engine_selection,
expression=dom_stability_js,
timeout_ms=timeout_ms,
)
diff --git a/tests/unit/test_page_evaluate_engine_classification.py b/tests/unit/test_page_evaluate_engine_classification.py
new file mode 100644
index 000000000..c5330e058
--- /dev/null
+++ b/tests/unit/test_page_evaluate_engine_classification.py
@@ -0,0 +1,276 @@
+"""``SkyvernFrame`` evaluate / navigation-context-loss recovery must key on the per-run selected
+browser engine's error family, not a hard-coded stock-Playwright ``except``. A run pinned to a
+non-Playwright engine must recover from ITS native context-loss error and settle-noise, while a
+foreign error (including another engine's error, or an unrelated exception) still propagates and
+cancellation is never swallowed. With no engine pinned, the exact stock-Playwright behavior is
+preserved.
+
+These stay driver-agnostic: they pin fake engine selections and script the evaluate callable, so
+they hold on an image shipping only stock Playwright.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+from playwright.async_api import Error as PlaywrightError
+
+from skyvern.exceptions import SkyvernPageAnalysisTimeout
+from skyvern.webeye.browser_engine import BrowserEngineMetadata, BrowserEngineSelection
+from skyvern.webeye.utils.page import JS_FUNCTION_DEFS, SkyvernFrame, _wait_for_navigation_settle
+
+_NAV_LOST = "Execution context was destroyed"
+
+
+class _EngineAError(Exception):
+ pass
+
+
+class _EngineATimeout(_EngineAError):
+ pass
+
+
+async def _never_start(): # pragma: no cover - never awaited in these tests
+ raise AssertionError("start_driver must not be called")
+
+
+def _selection(name: str, error_type: type[BaseException], timeout_type: type[BaseException]) -> BrowserEngineSelection:
+ return BrowserEngineSelection(
+ name=name,
+ start_driver=_never_start,
+ error_type=error_type,
+ timeout_error_type=timeout_type,
+ metadata=BrowserEngineMetadata(name=name, version="0.0.0"),
+ selection_reason="test",
+ )
+
+
+class _SettleFrame:
+ """Stands in for a Page/Frame during recovery: only ``wait_for_load_state`` is exercised."""
+
+ def __init__(self, on_settle: BaseException | None = None) -> None:
+ self.load_state_calls = 0
+ self._on_settle = on_settle
+
+ async def wait_for_load_state(self, state: str, timeout: float | None = None) -> None:
+ self.load_state_calls += 1
+ if self._on_settle is not None:
+ raise self._on_settle
+
+
+class _ScriptedEval:
+ def __init__(self, seq: list[object]) -> None:
+ self.seq = list(seq)
+ self.calls = 0
+
+ async def __call__(self) -> object:
+ item = self.seq[self.calls]
+ self.calls += 1
+ if isinstance(item, BaseException):
+ raise item
+ return item
+
+
+async def _run_evaluate(
+ seq: list[object],
+ engine_selection: BrowserEngineSelection | None,
+ *,
+ frame: _SettleFrame | None = None,
+ timeout_ms: float = 5000,
+) -> tuple[object, _ScriptedEval]:
+ scripted = _ScriptedEval(seq)
+ result = await SkyvernFrame._evaluate_expression(
+ frame=frame or _SettleFrame(), # type: ignore[arg-type]
+ expression=JS_FUNCTION_DEFS, # bootstrap expression skips the re-injection pass
+ evaluate_expression=scripted,
+ timeout_ms=timeout_ms,
+ engine_selection=engine_selection,
+ )
+ return result, scripted
+
+
+# -- evaluate/navigation-context-loss recovery -------------------------------------------------
+
+
+async def test_evaluate_recovers_on_selected_engine_native_context_loss() -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ result, scripted = await _run_evaluate([_EngineAError(_NAV_LOST), 42], sel)
+ assert result == 42
+ assert scripted.calls == 2
+
+
+async def test_evaluate_reraises_selected_engine_error_that_is_not_context_loss() -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ with pytest.raises(_EngineAError):
+ await _run_evaluate([_EngineAError("some unrelated boom")], sel)
+
+
+async def test_evaluate_propagates_foreign_error_even_when_message_is_context_loss() -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ # A stock-Playwright error under a non-Playwright selection is foreign: it must propagate raw,
+ # never be routed into engine-a's recovery, even though its message matches the recovery text.
+ with pytest.raises(PlaywrightError):
+ await _run_evaluate([PlaywrightError(_NAV_LOST)], sel)
+
+
+async def test_evaluate_retries_multiple_selected_engine_context_losses() -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ result, scripted = await _run_evaluate(
+ [_EngineAError(_NAV_LOST), _EngineAError("Cannot find context with specified id"), 99],
+ sel,
+ )
+ assert result == 99
+ assert scripted.calls == 3
+
+
+async def test_evaluate_recovers_on_stock_context_loss_when_selection_missing() -> None:
+ result, scripted = await _run_evaluate([PlaywrightError(_NAV_LOST), 7], None)
+ assert result == 7
+ assert scripted.calls == 2
+
+
+async def test_evaluate_reraises_stock_error_that_is_not_context_loss() -> None:
+ with pytest.raises(PlaywrightError):
+ await _run_evaluate([PlaywrightError("boom")], None)
+
+
+async def test_evaluate_propagates_nonplaywright_error_under_stock_selection() -> None:
+ with pytest.raises(_EngineAError):
+ await _run_evaluate([_EngineAError(_NAV_LOST)], None)
+
+
+async def test_evaluate_recovers_on_runtime_error_context_loss_regardless_of_engine() -> None:
+ for sel in (None, _selection("engine-a", _EngineAError, _EngineATimeout)):
+ result, scripted = await _run_evaluate([RuntimeError(_NAV_LOST), 5], sel)
+ assert result == 5
+ assert scripted.calls == 2
+
+
+async def test_evaluate_maps_asyncio_timeout_to_page_analysis_timeout() -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+
+ async def slow() -> object:
+ await asyncio.sleep(10)
+ return None
+
+ with pytest.raises(SkyvernPageAnalysisTimeout):
+ await SkyvernFrame._evaluate_expression(
+ frame=_SettleFrame(), # type: ignore[arg-type]
+ expression=JS_FUNCTION_DEFS,
+ evaluate_expression=slow,
+ timeout_ms=10,
+ engine_selection=sel,
+ )
+
+
+# -- navigation settle noise-tolerance ---------------------------------------------------------
+
+
+async def test_navigation_settle_swallows_selected_engine_error() -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ frame = _SettleFrame(on_settle=_EngineAError("still navigating"))
+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=sel) # type: ignore[arg-type]
+ assert frame.load_state_calls == 1
+
+
+async def test_navigation_settle_propagates_foreign_error() -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ frame = _SettleFrame(on_settle=PlaywrightError("still navigating"))
+ with pytest.raises(PlaywrightError):
+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=sel) # type: ignore[arg-type]
+
+
+async def test_navigation_settle_swallows_stock_playwright_error_when_selection_missing() -> None:
+ frame = _SettleFrame(on_settle=PlaywrightError("still navigating"))
+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=None) # type: ignore[arg-type]
+ assert frame.load_state_calls == 1
+
+
+async def test_navigation_settle_propagates_nonplaywright_error_under_stock() -> None:
+ frame = _SettleFrame(on_settle=_EngineAError("still navigating"))
+ with pytest.raises(_EngineAError):
+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=None) # type: ignore[arg-type]
+
+
+async def test_navigation_settle_propagates_cancellation() -> None:
+ frame = _SettleFrame(on_settle=asyncio.CancelledError())
+ with pytest.raises(asyncio.CancelledError):
+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=None) # type: ignore[arg-type]
+
+
+# -- instance threading ------------------------------------------------------------------------
+
+
+def test_skyvern_frame_stores_engine_selection_default_none() -> None:
+ assert SkyvernFrame(frame=object()).engine_selection is None # type: ignore[arg-type]
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ assert SkyvernFrame(frame=object(), engine_selection=sel).engine_selection is sel # type: ignore[arg-type]
+
+
+async def test_instance_evaluate_forwards_engine_selection(monkeypatch: pytest.MonkeyPatch) -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ captured: dict[str, object] = {}
+
+ async def fake_eval_expr(
+ *,
+ frame: object,
+ expression: str,
+ evaluate_expression: object,
+ timeout_ms: float,
+ engine_selection: object = None,
+ ) -> str:
+ captured["engine_selection"] = engine_selection
+ return "ok"
+
+ monkeypatch.setattr(SkyvernFrame, "_evaluate_expression", staticmethod(fake_eval_expr))
+ sf = SkyvernFrame(frame=object(), engine_selection=sel) # type: ignore[arg-type]
+ result = await sf.get_scroll_x_y()
+ assert result == "ok"
+ assert captured["engine_selection"] is sel
+
+
+async def test_instance_evaluate_expression_forwards_engine_selection(monkeypatch: pytest.MonkeyPatch) -> None:
+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)
+ captured: dict[str, object] = {}
+
+ async def fake_eval_expr(
+ *,
+ frame: object,
+ expression: str,
+ evaluate_expression: object,
+ timeout_ms: float,
+ engine_selection: object = None,
+ ) -> bool:
+ captured["engine_selection"] = engine_selection
+ return True
+
+ monkeypatch.setattr(SkyvernFrame, "_evaluate_expression", staticmethod(fake_eval_expr))
+
+ class _Loc:
+ async def count(self) -> int:
+ return 1
+
+ async def evaluate(self, _js: str) -> bool:
+ return True
+
+ sf = SkyvernFrame(frame=object(), engine_selection=sel) # type: ignore[arg-type]
+ result = await sf.get_element_visible(_Loc()) # type: ignore[arg-type]
+ assert result is True
+ assert captured["engine_selection"] is sel