Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[SKY-12007] S2: structural page-vs-frame predicate + browser-object identity AST guard (#7476)
LawyZheng230b1e6a75cb
diff --git a/skyvern/webeye/browser_object_predicates.py b/skyvern/webeye/browser_object_predicates.pynew file mode 100644index 000000000..edfab02ad--- /dev/null+++ b/skyvern/webeye/browser_object_predicates.py@@ -0,0 +1,68 @@+"""Structural predicates over browser page/frame objects.++Skyvern runs on stock Playwright and Patchright today, with room for further+engine adapters (e.g. a Rustwright candidate) behind the same call sites. Each+engine ships its own ``Page`` / ``Frame`` classes with distinct identities, so+``isinstance(obj, Page)`` really asks "is this *that package's* Page" -- it+silently returns ``False`` for an equivalent object produced by a different+engine and forces a concrete driver import at the call site.++These predicates answer the actual question -- "does this object behave like a+top-level page, or like a subframe?" -- from structural capability alone, naming+no driver class. They also hold for ``MagicMock(spec=Page)`` /+``MagicMock(spec=Frame)`` fixtures, which expose exactly the spec'd attributes.+"""++from __future__ import annotations++from typing import Any, Protocol, TypeGuard++# Every member the ``PageLike`` Protocol promises a caller may dereference once+# ``is_page_like`` holds; the runtime check must verify all of them or the+# ``TypeGuard`` is unsound. ``main_frame`` + ``bring_to_front`` are also the+# page-vs-frame discriminators (a subframe exposes neither), and ``context`` is+# page-only; ``evaluate`` is shared with frames but is still dereferenced on a+# narrowed page, so it is required too. Page-only across the Playwright/Patchright+# surface and any capability-equivalent future adapter.+_REQUIRED_PAGE_CAPABILITIES = ("main_frame", "context", "bring_to_front", "evaluate")++# A subframe reports the page and parent frame that own it; a top-level page+# exposes neither. Used as a negative guard so an object that happens to carry a+# page capability is still rejected when it is really a frame.+_FRAME_ONLY_CAPABILITIES = ("page", "parent_frame")+++class PageLike(Protocol):+ """Structural page surface a caller may rely on once ``is_page_like`` holds.++ Declares only the members Skyvern dereferences on a narrowed page, so the+ ``TypeGuard`` restores static narrowing (e.g. ``page.context``) without+ naming a concrete engine ``Page`` -- an alternate-engine object satisfies it+ by capability, not by nominal identity.+ """++ @property+ def main_frame(self) -> Any: ...++ @property+ def context(self) -> Any: ...++ def bring_to_front(self) -> Any: ...++ def evaluate(self, expression: str, arg: Any = ...) -> Any: ...+++def is_page_like(obj: object) -> TypeGuard[PageLike]:+ """True if ``obj`` structurally presents as a top-level browser page.++ Engine-neutral stand-in for ``isinstance(obj, Page)`` when the only thing+ that matters is page-vs-frame. Requires every capability the ``PageLike``+ Protocol promises to be present and every frame-only capability to be absent,+ so a real ``Frame`` (or a ``MagicMock(spec=Frame)``) is rejected even though+ it shares most of the page surface, while a page from any engine is accepted.+ """+ if not all(hasattr(obj, capability) for capability in _REQUIRED_PAGE_CAPABILITIES):+ return False+ if any(hasattr(obj, capability) for capability in _FRAME_ONLY_CAPABILITIES):+ return False+ return Truediff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.pyindex 5aebb5945..22b0c0a39 100644--- a/skyvern/webeye/utils/page.py+++ b/skyvern/webeye/utils/page.py@@ -21,6 +21,7 @@ from skyvern.exceptions import FailedToTakeScreenshotfrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.settings_manager import SettingsManagerfrom skyvern.forge.sdk.trace import apply_context_attrs, traced+from skyvern.webeye.browser_object_predicates import is_page_likefrom skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefixif TYPE_CHECKING:@@ -104,9 +105,13 @@ async def _dispatch_evaluate(frame: Page | Frame, expression: str, arg: Any | No# Page + prefix + JSON-safe arg → main-world hook (preserves the marker).# Iframe Frames and non-JSON args fall back to per-frame evaluate so iframe# contexts and Playwright handle-marshalling keep working.- if not isinstance(frame, Page):+ if not is_page_like(frame):return await frame.evaluate(expression=expression, arg=arg)- if get_main_world_prefix(frame.context) is None:+ context = frame.context+ # A page whose context is None (an engine's pre-attach/edge state) can't key+ # the prefix WeakKeyDictionary — get_main_world_prefix(None) would raise+ # instead of returning the no-prefix fallback, so take direct evaluate here.+ if context is None or get_main_world_prefix(context) is None:return await frame.evaluate(expression=expression, arg=arg)if arg is not None and not _is_json_inlinable(arg):return await frame.evaluate(expression=expression, arg=arg)diff --git a/tests/unit/test_browser_object_identity_guard.py b/tests/unit/test_browser_object_identity_guard.pynew file mode 100644index 000000000..57965a267--- /dev/null+++ b/tests/unit/test_browser_object_identity_guard.py@@ -0,0 +1,296 @@+"""AST guard against package-native browser-object identity checks in production.++Skyvern selects among several browser engines whose ``Page`` / ``Frame`` /+``Browser`` / ... classes have distinct identities. A runtime ``isinstance`` /+``issubclass`` / ``type(x) is`` check against one driver's object class silently+misclassifies an equivalent object from a different engine. This guard walks+production ``skyvern/**`` and ``cloud/**`` and fails when a new such check+appears; the structural predicates in ``skyvern.webeye.browser_object_predicates``+are the sanctioned alternative.++Deliberately out of scope (so this does not pretend the later exception-catch+migration is finished, and does not raise false positives):+ * annotations, ``TYPE_CHECKING`` imports, and ``cast(...)`` -- type-only usage,+ never a runtime branch;+ * ``except <native error>`` catches and identity checks against driver *error*+ classes (Error/TimeoutError/TargetClosedError) -- that is a separate tranche;+ * classes named Page/Frame/Response/... imported from non-driver packages+ (fastapi, starlette, httpx, ...), resolved by import origin.+"""++from __future__ import annotations++import ast+from dataclasses import dataclass+from pathlib import Path++import pytest++# Import roots of the engines whose object classes this guard polices. A new+# engine adapter that ships its own Page/Frame-alike classes under a distinct+# import root must be registered here, or identity checks against it slip past.+_BROWSER_DRIVER_ROOTS = ("playwright", "patchright", "rustwright")++# Browser *object* classes whose identity is engine-specific. Driver *error*+# classes (Error, TimeoutError, TargetClosedError, ...) are intentionally absent:+# migrating those catches is a separate tranche, not this guard's concern.+_BROWSER_OBJECT_CLASSES = frozenset(+ {+ "Page",+ "Frame",+ "FrameLocator",+ "Locator",+ "ElementHandle",+ "JSHandle",+ "Browser",+ "BrowserContext",+ "BrowserType",+ "CDPSession",+ "Download",+ "Request",+ "Response",+ "Route",+ "WebSocket",+ "WebSocketRoute",+ "Worker",+ "Dialog",+ "ConsoleMessage",+ "FileChooser",+ "Video",+ "Keyboard",+ "Mouse",+ "Touchscreen",+ "Accessibility",+ "Clock",+ "Selectors",+ "Tracing",+ "Playwright",+ }+)+++@dataclass(frozen=True)+class Violation:+ filename: str+ lineno: int+ class_name: str+ check: str+++def _module_root_is_driver(module: str | None) -> bool:+ if not module:+ return False+ return module.split(".", 1)[0] in _BROWSER_DRIVER_ROOTS+++def _dotted_path(node: ast.expr) -> str | None:+ parts: list[str] = []+ while isinstance(node, ast.Attribute):+ parts.append(node.attr)+ node = node.value+ if not isinstance(node, ast.Name):+ return None+ parts.append(node.id)+ parts.reverse()+ return ".".join(parts)+++class _IdentityCheckVisitor(ast.NodeVisitor):+ def __init__(self, filename: str) -> None:+ self.filename = filename+ self.violations: list[Violation] = []+ # local name -> originating driver object-class name (e.g. "P" -> "Frame")+ self._driver_object_names: dict[str, str] = {}+ # local module alias -> driver module (e.g. "pw" -> "playwright.async_api")+ self._driver_module_aliases: dict[str, str] = {}++ def visit_ImportFrom(self, node: ast.ImportFrom) -> None:+ if _module_root_is_driver(node.module):+ for alias in node.names:+ if alias.name in _BROWSER_OBJECT_CLASSES:+ self._driver_object_names[alias.asname or alias.name] = alias.name+ self.generic_visit(node)++ def visit_Import(self, node: ast.Import) -> None:+ for alias in node.names:+ if _module_root_is_driver(alias.name):+ self._driver_module_aliases[alias.asname or alias.name] = alias.name+ self.generic_visit(node)++ def _module_expr_is_driver(self, node: ast.expr) -> bool:+ # ``module_alias.Page`` where the module part may be a single alias+ # (``pw``) or a dotted package path (``playwright.async_api``). Match the+ # rendered path against a registered driver import, resolving by origin+ # (never by the ``Page`` text): an exact hit, or a submodule of an+ # imported driver root (``import playwright`` + ``playwright.async_api.Page``).+ path = _dotted_path(node)+ if path is None:+ return False+ for alias in self._driver_module_aliases:+ if path == alias or path.startswith(alias + "."):+ return True+ return False++ def _resolve(self, node: ast.expr) -> str | None:+ if isinstance(node, ast.Name):+ return self._driver_object_names.get(node.id)+ if isinstance(node, ast.Attribute) and node.attr in _BROWSER_OBJECT_CLASSES:+ if self._module_expr_is_driver(node.value):+ return node.attr+ return None++ def _record(self, node: ast.expr, check: str) -> None:+ targets = node.elts if isinstance(node, ast.Tuple) else [node]+ for target in targets:+ class_name = self._resolve(target)+ if class_name is not None:+ self.violations.append(Violation(self.filename, target.lineno, class_name, check))++ def visit_Call(self, node: ast.Call) -> None:+ if isinstance(node.func, ast.Name) and node.func.id in ("isinstance", "issubclass") and len(node.args) >= 2:+ self._record(node.args[1], node.func.id)+ self.generic_visit(node)++ def visit_Compare(self, node: ast.Compare) -> None:+ left = node.left+ is_type_call = isinstance(left, ast.Call) and isinstance(left.func, ast.Name) and left.func.id == "type"+ if is_type_call:+ for op, comparator in zip(node.ops, node.comparators):+ if isinstance(op, (ast.Is, ast.IsNot, ast.Eq, ast.NotEq)):+ self._record(comparator, "type-is")+ self.generic_visit(node)+++def find_browser_object_identity_checks(source: str, filename: str = "<test>") -> list[Violation]:+ visitor = _IdentityCheckVisitor(filename)+ visitor.visit(ast.parse(source))+ return visitor.violations+++# --- detector unit tests (synthetic sources) --------------------------------+++def test_flags_isinstance_against_driver_page() -> None:+ src = "from playwright.async_api import Page\nx = isinstance(obj, Page)\n"+ v = find_browser_object_identity_checks(src)+ assert [(x.class_name, x.check) for x in v] == [("Page", "isinstance")]+++def test_flags_aliased_import() -> None:+ src = "from patchright.async_api import Frame as F\nx = isinstance(obj, F)\n"+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Frame"]+++def test_flags_each_member_of_isinstance_tuple() -> None:+ src = "from playwright.async_api import Frame, Page\nx = isinstance(obj, (Page, Frame))\n"+ assert sorted(x.class_name for x in find_browser_object_identity_checks(src)) == ["Frame", "Page"]+++def test_flags_issubclass_and_type_is() -> None:+ src = "from rustwright import Browser\na = issubclass(cls, Browser)\nb = type(obj) is Browser\n"+ checks = sorted(x.check for x in find_browser_object_identity_checks(src))+ assert checks == ["issubclass", "type-is"]+++def test_flags_module_attribute_access() -> None:+ src = "import playwright.async_api as pw\nx = isinstance(obj, pw.Page)\n"+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Page"]+++def test_flags_unaliased_dotted_import_attribute_access() -> None:+ src = "import playwright.async_api\nx = isinstance(obj, playwright.async_api.Page)\n"+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Page"]+++def test_flags_dotted_attribute_access_under_bare_driver_root_import() -> None:+ src = "import playwright\nx = isinstance(obj, playwright.async_api.Frame)\n"+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Frame"]+++def test_flags_unaliased_dotted_import_for_other_driver_root() -> None:+ src = "import patchright.async_api\nx = issubclass(cls, patchright.async_api.Browser)\n"+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Browser"]+++def test_ignores_dotted_attribute_access_from_non_driver_package() -> None:+ src = "import fastapi.responses\nx = isinstance(summary, fastapi.responses.Response)\n"+ assert find_browser_object_identity_checks(src) == []+++def test_ignores_same_named_class_from_non_driver_package() -> None:+ src = "from fastapi import Response\nx = isinstance(summary, Response)\n"+ assert find_browser_object_identity_checks(src) == []+++def test_ignores_driver_error_classes() -> None:+ src = "from playwright.async_api import TimeoutError as PWTimeout\nx = isinstance(exc, PWTimeout)\n"+ assert find_browser_object_identity_checks(src) == []+++def test_ignores_annotations_and_cast() -> None:+ src = (+ "from typing import cast\n"+ "from playwright.async_api import Page\n"+ "def f(x: Page) -> Page:\n"+ " y: Page = x\n"+ " return cast(Page, y)\n"+ )+ assert find_browser_object_identity_checks(src) == []+++def test_ignores_type_checking_import_used_only_in_annotation() -> None:+ src = (+ "from typing import TYPE_CHECKING\n"+ "if TYPE_CHECKING:\n"+ " from playwright.async_api import Page\n"+ "def f(x: 'Page') -> None:\n"+ " return None\n"+ )+ assert find_browser_object_identity_checks(src) == []+++# --- repo enforcement -------------------------------------------------------+++def _production_python_files() -> list[Path]:+ repo_root = Path(__file__).resolve().parents[2]+ files: list[Path] = []+ for package in ("skyvern", "cloud"):+ root = repo_root / package+ if not root.is_dir():+ continue+ for path in root.rglob("*.py"):+ parts = set(path.parts)+ if "tests" in parts or path.name.startswith("test_") or "__pycache__" in parts:+ continue+ files.append(path)+ return files+++def test_no_browser_object_identity_checks_in_production() -> None:+ offenders: list[Violation] = []+ for path in _production_python_files():+ try:+ tree_src = path.read_text(encoding="utf-8")+ except (OSError, UnicodeDecodeError):+ continue+ try:+ offenders.extend(find_browser_object_identity_checks(tree_src, str(path)))+ except SyntaxError:+ continue+ assert not offenders, (+ "Package-native browser-object identity checks found. Use the structural "+ "predicates in skyvern.webeye.browser_object_predicates (e.g. is_page_like) "+ "instead of isinstance/issubclass/type() against a driver's object class:\n"+ + "\n".join(f" {o.filename}:{o.lineno} {o.check}({o.class_name})" for o in offenders)+ )+++def test_repo_scan_finds_at_least_the_production_tree() -> None:+ # Guard against the enforcement test silently passing because the scan walked+ # nothing (wrong root, glob typo). skyvern/** must always yield files.+ assert len(_production_python_files()) > 100+++if __name__ == "__main__":+ raise SystemExit(pytest.main([__file__, "-v"]))diff --git a/tests/unit/webeye/test_browser_object_predicates.py b/tests/unit/webeye/test_browser_object_predicates.pynew file mode 100644index 000000000..a70496277--- /dev/null+++ b/tests/unit/webeye/test_browser_object_predicates.py@@ -0,0 +1,77 @@+from __future__ import annotations++from unittest.mock import MagicMock++from playwright.async_api import Frame, Page++from skyvern.webeye.browser_object_predicates import is_page_like+++def test_real_page_class_instance_is_page_like() -> None:+ page = MagicMock(spec=Page)+ assert is_page_like(page) is True+++def test_real_frame_class_instance_is_not_page_like() -> None:+ frame = MagicMock(spec=Frame)+ assert is_page_like(frame) is False+++def test_unrelated_object_is_not_page_like() -> None:+ assert is_page_like(object()) is False+ assert is_page_like(ValueError("boom")) is False+ assert is_page_like(None) is False+++def test_matches_isinstance_page_across_representative_objects() -> None:+ # The predicate must be a drop-in for isinstance(obj, Page) at the one+ # migrated call site: identical verdict for a Page, a Frame, and their+ # spec'd mocks (which back the existing suite).+ for obj in (MagicMock(spec=Page), MagicMock(spec=Frame), object(), "not-a-page"):+ assert is_page_like(obj) == isinstance(obj, Page)+++def test_page_like_requires_all_page_only_capabilities() -> None:+ # A frame-tree owner exposes main_frame + bring_to_front; missing either+ # marker is not enough to be a page.+ only_main_frame = MagicMock(spec=["main_frame", "url", "evaluate"])+ assert is_page_like(only_main_frame) is False+++def test_page_like_rejects_object_carrying_frame_ownership_markers() -> None:+ # Even a duck object that exposes every page capability is rejected when it+ # also reports a parent page/frame -- that is a subframe, not a page.+ frame_shaped = MagicMock(spec=["main_frame", "context", "bring_to_front", "evaluate", "page", "parent_frame"])+ assert is_page_like(frame_shaped) is False+++def test_page_like_rejects_object_missing_context() -> None:+ # context backs the PageLike TypeGuard (the call site dereferences+ # frame.context), so an object without it must not narrow to PageLike even+ # with both page-vs-frame discriminators present.+ no_context = MagicMock(spec=["main_frame", "bring_to_front", "evaluate"])+ assert is_page_like(no_context) is False+++def test_page_like_rejects_object_missing_evaluate() -> None:+ # evaluate is a PageLike member dereferenced on the narrowed page; missing it+ # means the object does not fully satisfy the contract.+ no_evaluate = MagicMock(spec=["main_frame", "context", "bring_to_front"])+ assert is_page_like(no_evaluate) is False+++def test_engine_neutral_page_from_a_distinct_class_is_page_like() -> None:+ # A second engine's Page has a distinct class identity, so isinstance against+ # a concrete driver's Page would return False for it; the structural+ # predicate accepts it because it genuinely presents the full PageLike+ # capability surface (main_frame, context, bring_to_front, evaluate).+ class OtherEnginePage:+ main_frame = object()+ context = object()++ def bring_to_front(self) -> None: ...++ def evaluate(self, expression: str, arg: object = None) -> None: ...++ assert isinstance(OtherEnginePage(), Page) is False+ assert is_page_like(OtherEnginePage()) is Truediff --git a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py b/tests/unit/webeye/test_skyvern_frame_main_world_routing.pyindex 4dee9c234..719f6b346 100644--- a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py+++ b/tests/unit/webeye/test_skyvern_frame_main_world_routing.py@@ -94,6 +94,22 @@ class TestSkyvernFrameEvaluateRouting:page.evaluate.assert_awaited_once_with(expression="(el) => el.blur()", arg=element_handle_like)page.context.new_cdp_session.assert_not_awaited() # type: ignore[attr-defined]+ @pytest.mark.asyncio+ async def test_page_like_with_none_context_falls_back_to_direct_evaluate(self) -> None:+ """A page-like object whose context is None (a future engine's pre-attach+ state) must take direct frame.evaluate and never reach the prefix+ WeakKeyDictionary, whose get(None) would raise TypeError."""+ page = MagicMock(spec=Page)+ page.context = None+ page.evaluate = AsyncMock(return_value="page-evaluate-result")++ with patch("skyvern.webeye.utils.page.get_main_world_prefix") as get_prefix:+ result = await SkyvernFrame.evaluate(page, "() => 1")++ assert result == "page-evaluate-result"+ page.evaluate.assert_awaited_once_with(expression="() => 1", arg=None)+ get_prefix.assert_not_called()+@pytest.mark.asyncioasync def test_frame_target_never_routes_through_runtime_evaluate(self) -> None:"""Iframe Frames have their own world; page-level Runtime.evaluate lands elsewhere."""