Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Route page.py evaluate/context-loss recovery through the selected browser engine (RUS-5) [R4a] (#7811)
LawyZhenge5a37c922ea0
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex dced0c5c7..ee8738e6f 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -24,6 +24,7 @@ from skyvern.forge.sdk.browser_action_preflight import policy_observation_enablefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.trace import apply_context_attrs, traced+from skyvern.webeye.browser_engine import BrowserEngineSelectionfrom skyvern.webeye.browser_object_predicates import is_page_likefrom skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefix@@ -139,6 +140,13 @@ _NAVIGATION_RECOVERY_MAX_ATTEMPTS = 4_NAVIGATION_SETTLE_TIMEOUT_MS = 3000+def _is_engine_error(exc: BaseException, engine_selection: BrowserEngineSelection | None) -> bool:+ """Whether ``exc`` belongs to THIS run's selected browser-engine error family. Falls back to the+ stock-Playwright identity when no engine is pinned, so an unmigrated caller keeps exact stock+ behavior; a foreign engine's error (or an unrelated exception) is rejected and left to propagate."""+ return engine_selection.is_engine_error(exc) if engine_selection is not None else isinstance(exc, PlaywrightError)++def _is_navigation_context_lost(error_msg: str) -> bool:if "Execution context was destroyed" in error_msg:return True@@ -174,12 +182,18 @@ async def _dispatch_evaluate(frame: Page | Frame, expression: str, arg: Any | Noreturn await evaluate_in_main_world(frame, expression, arg)-async def _wait_for_navigation_settle(frame: Page | Frame, timeout_ms: float) -> None:+async def _wait_for_navigation_settle(+ frame: Page | Frame,+ timeout_ms: float,+ engine_selection: BrowserEngineSelection | None = None,+) -> None:if timeout_ms <= 0:returntry:await frame.wait_for_load_state("networkidle", timeout=timeout_ms)- except PlaywrightError:+ except Exception as e:+ if not _is_engine_error(e, engine_selection):+ raisereturn@@ -604,12 +618,15 @@ def pop_destination_facts(nodes: object) -> dict[str, dict]:class SkyvernFrame:+ engine_selection: BrowserEngineSelection | None = None+@staticmethodasync def evaluate(frame: Page | Frame,expression: str,arg: Any | None = None,timeout_ms: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,+ engine_selection: BrowserEngineSelection | None = None,) -> Any:async def evaluate_expression() -> Any:return await _dispatch_evaluate(frame, expression, arg)@@ -619,6 +636,7 @@ class SkyvernFrame:expression=expression,evaluate_expression=evaluate_expression,timeout_ms=timeout_ms,+ engine_selection=engine_selection,)@staticmethod@@ -627,11 +645,19 @@ class SkyvernFrame:expression: str,evaluate_expression: Callable[[], Awaitable[Any]],timeout_ms: float,+ engine_selection: BrowserEngineSelection | None = None,) -> Any:try:async with asyncio.timeout(timeout_ms / 1000):return await evaluate_expression()- except PlaywrightError as e:+ except asyncio.TimeoutError as error:+ # Re-raised and handled by the caller (scrape retries / failure classification),+ # so this is not the failure boundary; log without a traceback at warning.+ LOG.warning("Skyvern timed out trying to analyze the page", expression=expression)+ raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error+ except RuntimeError as e:+ # `evaluate_in_main_world` raises RuntimeError on Runtime.evaluate+ # exception payloads; only navigation-context-lost text recovers here.error_msg = str(e)if not _is_navigation_context_lost(error_msg):raise@@ -641,10 +667,14 @@ class SkyvernFrame:evaluate_expression=evaluate_expression,timeout_ms=timeout_ms,initial_error=error_msg,+ engine_selection=engine_selection,)- except RuntimeError as e:- # `evaluate_in_main_world` raises RuntimeError on Runtime.evaluate- # exception payloads; only navigation-context-lost text recovers here.+ except Exception as e:+ # A driver-native error from THIS run's engine (stock Playwright when no engine is+ # pinned). A foreign engine's error, or an unrelated exception, is not ours: re-raise it+ # unchanged. Cancellation (BaseException, not Exception) is never caught here.+ if not _is_engine_error(e, engine_selection):+ raiseerror_msg = str(e)if not _is_navigation_context_lost(error_msg):raise@@ -654,12 +684,8 @@ class SkyvernFrame:evaluate_expression=evaluate_expression,timeout_ms=timeout_ms,initial_error=error_msg,+ engine_selection=engine_selection,)- except asyncio.TimeoutError as error:- # Re-raised and handled by the caller (scrape retries / failure classification),- # so this is not the failure boundary; log without a traceback at warning.- LOG.warning("Skyvern timed out trying to analyze the page", expression=expression)- raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error@staticmethodasync def _evaluate_with_navigation_recovery(@@ -668,6 +694,7 @@ class SkyvernFrame:evaluate_expression: Callable[[], Awaitable[Any]],timeout_ms: float,initial_error: str,+ engine_selection: BrowserEngineSelection | None = None,) -> Any:# Multi-hop SSO/OIDC flows (especially response_mode=form_post) can destroy# the JS execution context several times in a row as the page auto-submits@@ -697,7 +724,7 @@ class SkyvernFrame:error=last_error_msg[:200],)settle_ms = min(_NAVIGATION_SETTLE_TIMEOUT_MS, _remaining_seconds() * 1000)- await _wait_for_navigation_settle(frame, timeout_ms=settle_ms)+ await _wait_for_navigation_settle(frame, timeout_ms=settle_ms, engine_selection=engine_selection)# The bootstrap call already IS the domUtils.js injection, so the retry below# re-injects it anyway; a separate injection pass would spend a second attempt@@ -721,7 +748,12 @@ class SkyvernFrame:expression=expression,)raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error- except (PlaywrightError, RuntimeError) as inject_err:+ except Exception as inject_err:+ # RuntimeError (main-world Runtime.evaluate payloads) is engine-agnostic; the+ # driver-native failure routes through the per-run engine identity. A foreign+ # error is re-raised; cancellation never reaches here.+ if not (isinstance(inject_err, RuntimeError) or _is_engine_error(inject_err, engine_selection)):+ raiselast_error_msg = str(inject_err)if attempt == _NAVIGATION_RECOVERY_MAX_ATTEMPTS or not _is_navigation_context_lost(last_error_msg):LOG.warning(@@ -744,7 +776,9 @@ class SkyvernFrame:except asyncio.TimeoutError as error:LOG.exception("Skyvern timed out on retry after JS context re-injection", expression=expression)raise SkyvernPageAnalysisTimeout("Skyvern timed out trying to analyze the page") from error- except (PlaywrightError, RuntimeError) as retry_err:+ except Exception as retry_err:+ if not (isinstance(retry_err, RuntimeError) or _is_engine_error(retry_err, engine_selection)):+ raiselast_error_msg = str(retry_err)if attempt == _NAVIGATION_RECOVERY_MAX_ATTEMPTS or not _is_navigation_context_lost(last_error_msg):raise@@ -1038,17 +1072,25 @@ class SkyvernFrame:return screenshots@classmethod- async def create_instance(cls, frame: Page | Frame) -> SkyvernFrame:- instance = cls(frame=frame)- await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS)+ async def create_instance(+ cls, frame: Page | Frame, engine_selection: BrowserEngineSelection | None = None+ ) -> SkyvernFrame:+ instance = cls(frame=frame, engine_selection=engine_selection)+ await cls.evaluate(frame=instance.frame, expression=JS_FUNCTION_DEFS, engine_selection=engine_selection)if SettingsManager.get_settings().ENABLE_EXP_ALL_TEXTUAL_ELEMENTS_INTERACTABLE:await instance.evaluate(- frame=instance.frame, expression="() => window.GlobalEnableAllTextualElements = true"+ frame=instance.frame,+ expression="() => window.GlobalEnableAllTextualElements = true",+ engine_selection=engine_selection,)return instance- def __init__(self, frame: Page | Frame) -> None:+ def __init__(self, frame: Page | Frame, engine_selection: BrowserEngineSelection | None = None) -> None:self.frame = frame+ # The logical run's pinned engine (None when the frame was created outside the per-run engine+ # seam), forwarded to every evaluate so navigation-context-loss recovery keys on THIS engine's+ # native error family; None preserves the exact stock-Playwright behavior.+ self.engine_selection = engine_selectiondef get_frame(self) -> Page | Frame:return self.frame@@ -1060,15 +1102,17 @@ class SkyvernFrame:async def get_scroll_x_y(self) -> tuple[int, int]:js_script = "() => getScrollXY()"- return await self.evaluate(frame=self.frame, expression=js_script)+ return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def get_scroll_width_and_height(self) -> tuple[int, int]:js_script = "() => getScrollWidthAndHeight()"- return await self.evaluate(frame=self.frame, expression=js_script)+ return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def scroll_to_x_y(self, x: int, y: int) -> None:js_script = "([x, y]) => scrollToXY(x, y)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=[x, y])+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[x, y]+ )async def safe_scroll_to_x_y(self, x: int, y: int) -> None:try:@@ -1080,25 +1124,38 @@ class SkyvernFrame:"""Scroll all ancestor containers (including nested ones with overflow-y: auto)so that the element is centered in the viewport."""js_script = "(element) => element.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'})"- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element+ )async def scroll_to_element_bottom(self, element: ElementHandle, page_by_page: bool = False) -> None:js_script = "([element, page_by_page]) => scrollToElementBottom(element, page_by_page)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=[element, page_by_page])+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element, page_by_page]+ )async def scroll_to_element_top(self, element: ElementHandle) -> None:js_script = "(element) => scrollToElementTop(element)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element+ )async def parse_element_from_html(self, frame: str, element: ElementHandle, interactable: bool) -> dict:js_script = "async ([frame, element, interactable]) => await buildElementObject(frame, element, interactable)"- parsed = await self.evaluate(frame=self.frame, expression=js_script, arg=[frame, element, interactable])+ parsed = await self.evaluate(+ frame=self.frame,+ engine_selection=self.engine_selection,+ expression=js_script,+ arg=[frame, element, interactable],+ )pop_destination_facts([parsed])return parsedasync def get_element_scrollable(self, element: ElementHandle) -> bool:js_script = "(element) => isScrollable(element)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element+ )async def get_element_visible(self, locator: Locator) -> bool:js_script = "(element) => isElementVisible(element) && !isHidden(element)"@@ -1110,6 +1167,7 @@ class SkyvernFrame:return await self._evaluate_expression(frame=self.frame,+ engine_selection=self.engine_selection,expression=js_script,evaluate_expression=evaluate_expression,timeout_ms=SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,@@ -1117,11 +1175,15 @@ class SkyvernFrame:async def get_disabled_from_style(self, element: ElementHandle) -> bool:js_script = "(element) => checkDisabledFromStyle(element)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element+ )async def get_blocking_element_id(self, element: ElementHandle) -> tuple[str, bool]:js_script = "(element) => getBlockElementUniqueID(element)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element+ )async def scroll_to_top(self, draw_boxes: bool, frame: str, frame_index: int) -> float:"""@@ -1133,6 +1195,7 @@ class SkyvernFrame:js_script = "async ([draw_boxes, frame, frame_index]) => await safeScrollToTop(draw_boxes, frame, frame_index)"scroll_y_px = await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression=js_script,timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,arg=[draw_boxes, frame, frame_index],@@ -1157,6 +1220,7 @@ class SkyvernFrame:js_script = "async ([draw_boxes, frame, frame_index, need_overlap]) => await scrollToNextPage(draw_boxes, frame, frame_index, need_overlap)"scroll_y_px = await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression=js_script,timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,arg=[draw_boxes, frame, frame_index, need_overlap],@@ -1177,6 +1241,7 @@ class SkyvernFrame:js_script = "() => removeBoundingBoxes()"await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression=js_script,timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,)@@ -1185,6 +1250,7 @@ class SkyvernFrame:js_script = "async ([frame, frame_index]) => await buildElementsAndDrawBoundingBoxes(frame, frame_index)"await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression=js_script,timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,arg=[frame, frame_index],@@ -1192,23 +1258,29 @@ class SkyvernFrame:async def is_window_scrollable(self) -> bool:js_script = "() => isWindowScrollable()"- return await self.evaluate(frame=self.frame, expression=js_script)+ return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def is_parent(self, parent: ElementHandle, child: ElementHandle) -> bool:js_script = "([parent, child]) => isParent(parent, child)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=[parent, child])+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[parent, child]+ )async def is_sibling(self, el1: ElementHandle, el2: ElementHandle) -> bool:js_script = "([el1, el2]) => isSibling(el1, el2)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=[el1, el2])+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[el1, el2]+ )async def has_ASP_client_control(self) -> bool:js_script = "() => hasASPClientControl()"- return await self.evaluate(frame=self.frame, expression=js_script)+ return await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def click_element_in_javascript(self, element: ElementHandle) -> None:js_script = "(element) => element.click()"- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element+ )async def read_autocomplete_option_identity(self, element: ElementHandle) -> dict[str, Any] | None:js_script = r"""@@ -1235,30 +1307,39 @@ class SkyvernFrame:return { index: optionNodes.indexOf(node), label };}"""- identity = await self.evaluate(frame=self.frame, expression=js_script, arg=element)+ identity = await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element+ )return identity if isinstance(identity, dict) else Noneasync def remove_target_attr(self, element: ElementHandle) -> None:js_script = "(element) => element.removeAttribute('target')"- return await self.evaluate(frame=self.frame, expression=js_script, arg=element)+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=element+ )async def get_select_options(self, element: ElementHandle) -> tuple[list, str]:js_script = "([element]) => getSelectOptions(element)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=[element])+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element]+ )async def get_element_dom_depth(self, element: ElementHandle) -> int:js_script = "([element]) => getElementDomDepth(element)"- return await self.evaluate(frame=self.frame, expression=js_script, arg=[element])+ return await self.evaluate(+ frame=self.frame, engine_selection=self.engine_selection, expression=js_script, arg=[element]+ )async def remove_all_unique_ids(self) -> None:js_script = "() => removeAllUniqueIds()"- await self.evaluate(frame=self.frame, expression=js_script)+ await self.evaluate(frame=self.frame, engine_selection=self.engine_selection, expression=js_script)async def _set_enriched_element_tree_flag(self) -> None:context = skyvern_context.current()enriched_enabled = bool(context and context.enriched_tree_enabled())await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression="([enabled]) => { window.GlobalEnableEnrichedElementTree = enabled; }",arg=[enriched_enabled],)@@ -1282,6 +1363,7 @@ class SkyvernFrame:js_script = "async ([frame_name, frame_index, must_included_tags, capture_destination_facts]) => await buildTreeFromBody(frame_name, frame_index, must_included_tags, capture_destination_facts)"elements, element_tree = await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression=js_script,timeout_ms=timeout_ms,arg=[frame_name, frame_index, must_included_tags, capture_destination_facts],@@ -1299,7 +1381,11 @@ class SkyvernFrame:await self._set_enriched_element_tree_flag()js_script = "async ([wait_until_finished]) => await getIncrementElements(wait_until_finished)"elements, element_tree = await self.evaluate(- frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[wait_until_finished]+ frame=self.frame,+ engine_selection=self.engine_selection,+ expression=js_script,+ timeout_ms=timeout_ms,+ arg=[wait_until_finished],)pop_destination_facts(elements)pop_destination_facts(element_tree)@@ -1316,7 +1402,11 @@ class SkyvernFrame:await self._set_enriched_element_tree_flag()js_script = "async ([starter, frame, full_tree]) => await buildElementTree(starter, frame, full_tree)"elements, element_tree = await self.evaluate(- frame=self.frame, expression=js_script, timeout_ms=timeout_ms, arg=[starter, frame, full_tree]+ frame=self.frame,+ engine_selection=self.engine_selection,+ expression=js_script,+ timeout_ms=timeout_ms,+ arg=[starter, frame, full_tree],)pop_destination_facts(elements)pop_destination_facts(element_tree)@@ -1354,6 +1444,7 @@ class SkyvernFrame:while True:is_finished = await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression="() => isAnimationFinished()",timeout_ms=timeout_ms,)@@ -1494,6 +1585,7 @@ class SkyvernFrame:while True:has_loading_indicator = await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression=loading_indicator_js,timeout_ms=timeout_ms,)@@ -1558,6 +1650,7 @@ class SkyvernFrame:await self.evaluate(frame=self.frame,+ engine_selection=self.engine_selection,expression=dom_stability_js,timeout_ms=timeout_ms,)diff --git a/tests/unit/test_page_evaluate_engine_classification.py b/tests/unit/test_page_evaluate_engine_classification.pynew file mode 100644index 000000000..c5330e058--- /dev/null+++ b/tests/unit/test_page_evaluate_engine_classification.py@@ -0,0 +1,276 @@+"""``SkyvernFrame`` evaluate / navigation-context-loss recovery must key on the per-run selected+browser engine's error family, not a hard-coded stock-Playwright ``except``. A run pinned to a+non-Playwright engine must recover from ITS native context-loss error and settle-noise, while a+foreign error (including another engine's error, or an unrelated exception) still propagates and+cancellation is never swallowed. With no engine pinned, the exact stock-Playwright behavior is+preserved.++These stay driver-agnostic: they pin fake engine selections and script the evaluate callable, so+they hold on an image shipping only stock Playwright.+"""++from __future__ import annotations++import asyncio++import pytest+from playwright.async_api import Error as PlaywrightError++from skyvern.exceptions import SkyvernPageAnalysisTimeout+from skyvern.webeye.browser_engine import BrowserEngineMetadata, BrowserEngineSelection+from skyvern.webeye.utils.page import JS_FUNCTION_DEFS, SkyvernFrame, _wait_for_navigation_settle++_NAV_LOST = "Execution context was destroyed"+++class _EngineAError(Exception):+ pass+++class _EngineATimeout(_EngineAError):+ pass+++async def _never_start(): # pragma: no cover - never awaited in these tests+ raise AssertionError("start_driver must not be called")+++def _selection(name: str, error_type: type[BaseException], timeout_type: type[BaseException]) -> BrowserEngineSelection:+ return BrowserEngineSelection(+ name=name,+ start_driver=_never_start,+ error_type=error_type,+ timeout_error_type=timeout_type,+ metadata=BrowserEngineMetadata(name=name, version="0.0.0"),+ selection_reason="test",+ )+++class _SettleFrame:+ """Stands in for a Page/Frame during recovery: only ``wait_for_load_state`` is exercised."""++ def __init__(self, on_settle: BaseException | None = None) -> None:+ self.load_state_calls = 0+ self._on_settle = on_settle++ async def wait_for_load_state(self, state: str, timeout: float | None = None) -> None:+ self.load_state_calls += 1+ if self._on_settle is not None:+ raise self._on_settle+++class _ScriptedEval:+ def __init__(self, seq: list[object]) -> None:+ self.seq = list(seq)+ self.calls = 0++ async def __call__(self) -> object:+ item = self.seq[self.calls]+ self.calls += 1+ if isinstance(item, BaseException):+ raise item+ return item+++async def _run_evaluate(+ seq: list[object],+ engine_selection: BrowserEngineSelection | None,+ *,+ frame: _SettleFrame | None = None,+ timeout_ms: float = 5000,+) -> tuple[object, _ScriptedEval]:+ scripted = _ScriptedEval(seq)+ result = await SkyvernFrame._evaluate_expression(+ frame=frame or _SettleFrame(), # type: ignore[arg-type]+ expression=JS_FUNCTION_DEFS, # bootstrap expression skips the re-injection pass+ evaluate_expression=scripted,+ timeout_ms=timeout_ms,+ engine_selection=engine_selection,+ )+ return result, scripted+++# -- evaluate/navigation-context-loss recovery -------------------------------------------------+++async def test_evaluate_recovers_on_selected_engine_native_context_loss() -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ result, scripted = await _run_evaluate([_EngineAError(_NAV_LOST), 42], sel)+ assert result == 42+ assert scripted.calls == 2+++async def test_evaluate_reraises_selected_engine_error_that_is_not_context_loss() -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ with pytest.raises(_EngineAError):+ await _run_evaluate([_EngineAError("some unrelated boom")], sel)+++async def test_evaluate_propagates_foreign_error_even_when_message_is_context_loss() -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ # A stock-Playwright error under a non-Playwright selection is foreign: it must propagate raw,+ # never be routed into engine-a's recovery, even though its message matches the recovery text.+ with pytest.raises(PlaywrightError):+ await _run_evaluate([PlaywrightError(_NAV_LOST)], sel)+++async def test_evaluate_retries_multiple_selected_engine_context_losses() -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ result, scripted = await _run_evaluate(+ [_EngineAError(_NAV_LOST), _EngineAError("Cannot find context with specified id"), 99],+ sel,+ )+ assert result == 99+ assert scripted.calls == 3+++async def test_evaluate_recovers_on_stock_context_loss_when_selection_missing() -> None:+ result, scripted = await _run_evaluate([PlaywrightError(_NAV_LOST), 7], None)+ assert result == 7+ assert scripted.calls == 2+++async def test_evaluate_reraises_stock_error_that_is_not_context_loss() -> None:+ with pytest.raises(PlaywrightError):+ await _run_evaluate([PlaywrightError("boom")], None)+++async def test_evaluate_propagates_nonplaywright_error_under_stock_selection() -> None:+ with pytest.raises(_EngineAError):+ await _run_evaluate([_EngineAError(_NAV_LOST)], None)+++async def test_evaluate_recovers_on_runtime_error_context_loss_regardless_of_engine() -> None:+ for sel in (None, _selection("engine-a", _EngineAError, _EngineATimeout)):+ result, scripted = await _run_evaluate([RuntimeError(_NAV_LOST), 5], sel)+ assert result == 5+ assert scripted.calls == 2+++async def test_evaluate_maps_asyncio_timeout_to_page_analysis_timeout() -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)++ async def slow() -> object:+ await asyncio.sleep(10)+ return None++ with pytest.raises(SkyvernPageAnalysisTimeout):+ await SkyvernFrame._evaluate_expression(+ frame=_SettleFrame(), # type: ignore[arg-type]+ expression=JS_FUNCTION_DEFS,+ evaluate_expression=slow,+ timeout_ms=10,+ engine_selection=sel,+ )+++# -- navigation settle noise-tolerance ---------------------------------------------------------+++async def test_navigation_settle_swallows_selected_engine_error() -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ frame = _SettleFrame(on_settle=_EngineAError("still navigating"))+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=sel) # type: ignore[arg-type]+ assert frame.load_state_calls == 1+++async def test_navigation_settle_propagates_foreign_error() -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ frame = _SettleFrame(on_settle=PlaywrightError("still navigating"))+ with pytest.raises(PlaywrightError):+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=sel) # type: ignore[arg-type]+++async def test_navigation_settle_swallows_stock_playwright_error_when_selection_missing() -> None:+ frame = _SettleFrame(on_settle=PlaywrightError("still navigating"))+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=None) # type: ignore[arg-type]+ assert frame.load_state_calls == 1+++async def test_navigation_settle_propagates_nonplaywright_error_under_stock() -> None:+ frame = _SettleFrame(on_settle=_EngineAError("still navigating"))+ with pytest.raises(_EngineAError):+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=None) # type: ignore[arg-type]+++async def test_navigation_settle_propagates_cancellation() -> None:+ frame = _SettleFrame(on_settle=asyncio.CancelledError())+ with pytest.raises(asyncio.CancelledError):+ await _wait_for_navigation_settle(frame, timeout_ms=100, engine_selection=None) # type: ignore[arg-type]+++# -- instance threading ------------------------------------------------------------------------+++def test_skyvern_frame_stores_engine_selection_default_none() -> None:+ assert SkyvernFrame(frame=object()).engine_selection is None # type: ignore[arg-type]+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ assert SkyvernFrame(frame=object(), engine_selection=sel).engine_selection is sel # type: ignore[arg-type]+++async def test_instance_evaluate_forwards_engine_selection(monkeypatch: pytest.MonkeyPatch) -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ captured: dict[str, object] = {}++ async def fake_eval_expr(+ *,+ frame: object,+ expression: str,+ evaluate_expression: object,+ timeout_ms: float,+ engine_selection: object = None,+ ) -> str:+ captured["engine_selection"] = engine_selection+ return "ok"++ monkeypatch.setattr(SkyvernFrame, "_evaluate_expression", staticmethod(fake_eval_expr))+ sf = SkyvernFrame(frame=object(), engine_selection=sel) # type: ignore[arg-type]+ result = await sf.get_scroll_x_y()+ assert result == "ok"+ assert captured["engine_selection"] is sel+++async def test_instance_evaluate_expression_forwards_engine_selection(monkeypatch: pytest.MonkeyPatch) -> None:+ sel = _selection("engine-a", _EngineAError, _EngineATimeout)+ captured: dict[str, object] = {}++ async def fake_eval_expr(+ *,+ frame: object,+ expression: str,+ evaluate_expression: object,+ timeout_ms: float,+ engine_selection: object = None,+ ) -> bool:+ captured["engine_selection"] = engine_selection+ return True++ monkeypatch.setattr(SkyvernFrame, "_evaluate_expression", staticmethod(fake_eval_expr))++ class _Loc:+ async def count(self) -> int:+ return 1++ async def evaluate(self, _js: str) -> bool:+ return True++ sf = SkyvernFrame(frame=object(), engine_selection=sel) # type: ignore[arg-type]+ result = await sf.get_element_visible(_Loc()) # type: ignore[arg-type]+ assert result is True+ assert captured["engine_selection"] is sel