Skip to content

Automate browser based workflows with AI

[SKY-12007] S2: structural page-vs-frame predicate + browser-object identity AST guard (#7476)

LawyZheng230b1e6a75cb
diff --git a/skyvern/webeye/browser_object_predicates.py b/skyvern/webeye/browser_object_predicates.py
new file mode 100644
index 000000000..edfab02ad
--- /dev/null
+++ b/skyvern/webeye/browser_object_predicates.py
@@ -0,0 +1,68 @@
+"""Structural predicates over browser page/frame objects.
+
+Skyvern runs on stock Playwright and Patchright today, with room for further
+engine adapters (e.g. a Rustwright candidate) behind the same call sites. Each
+engine ships its own ``Page`` / ``Frame`` classes with distinct identities, so
+``isinstance(obj, Page)`` really asks "is this *that package's* Page" -- it
+silently returns ``False`` for an equivalent object produced by a different
+engine and forces a concrete driver import at the call site.
+
+These predicates answer the actual question -- "does this object behave like a
+top-level page, or like a subframe?" -- from structural capability alone, naming
+no driver class. They also hold for ``MagicMock(spec=Page)`` /
+``MagicMock(spec=Frame)`` fixtures, which expose exactly the spec'd attributes.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Protocol, TypeGuard
+
+# Every member the ``PageLike`` Protocol promises a caller may dereference once
+# ``is_page_like`` holds; the runtime check must verify all of them or the
+# ``TypeGuard`` is unsound. ``main_frame`` + ``bring_to_front`` are also the
+# page-vs-frame discriminators (a subframe exposes neither), and ``context`` is
+# page-only; ``evaluate`` is shared with frames but is still dereferenced on a
+# narrowed page, so it is required too. Page-only across the Playwright/Patchright
+# surface and any capability-equivalent future adapter.
+_REQUIRED_PAGE_CAPABILITIES = ("main_frame", "context", "bring_to_front", "evaluate")
+
+# A subframe reports the page and parent frame that own it; a top-level page
+# exposes neither. Used as a negative guard so an object that happens to carry a
+# page capability is still rejected when it is really a frame.
+_FRAME_ONLY_CAPABILITIES = ("page", "parent_frame")
+
+
+class PageLike(Protocol):
+ """Structural page surface a caller may rely on once ``is_page_like`` holds.
+
+ Declares only the members Skyvern dereferences on a narrowed page, so the
+ ``TypeGuard`` restores static narrowing (e.g. ``page.context``) without
+ naming a concrete engine ``Page`` -- an alternate-engine object satisfies it
+ by capability, not by nominal identity.
+ """
+
+ @property
+ def main_frame(self) -> Any: ...
+
+ @property
+ def context(self) -> Any: ...
+
+ def bring_to_front(self) -> Any: ...
+
+ def evaluate(self, expression: str, arg: Any = ...) -> Any: ...
+
+
+def is_page_like(obj: object) -> TypeGuard[PageLike]:
+ """True if ``obj`` structurally presents as a top-level browser page.
+
+ Engine-neutral stand-in for ``isinstance(obj, Page)`` when the only thing
+ that matters is page-vs-frame. Requires every capability the ``PageLike``
+ Protocol promises to be present and every frame-only capability to be absent,
+ so a real ``Frame`` (or a ``MagicMock(spec=Frame)``) is rejected even though
+ it shares most of the page surface, while a page from any engine is accepted.
+ """
+ if not all(hasattr(obj, capability) for capability in _REQUIRED_PAGE_CAPABILITIES):
+ return False
+ if any(hasattr(obj, capability) for capability in _FRAME_ONLY_CAPABILITIES):
+ return False
+ return True
diff --git a/skyvern/webeye/utils/page.py b/skyvern/webeye/utils/page.py
index 5aebb5945..22b0c0a39 100644
--- a/skyvern/webeye/utils/page.py
+++ b/skyvern/webeye/utils/page.py
@@ -21,6 +21,7 @@ from skyvern.exceptions import FailedToTakeScreenshot
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import apply_context_attrs, traced
+from skyvern.webeye.browser_object_predicates import is_page_like
from skyvern.webeye.main_world_eval import evaluate_in_main_world, get_main_world_prefix
if TYPE_CHECKING:
@@ -104,9 +105,13 @@ async def _dispatch_evaluate(frame: Page | Frame, expression: str, arg: Any | No
# Page + prefix + JSON-safe arg → main-world hook (preserves the marker).
# Iframe Frames and non-JSON args fall back to per-frame evaluate so iframe
# contexts and Playwright handle-marshalling keep working.
- if not isinstance(frame, Page):
+ if not is_page_like(frame):
return await frame.evaluate(expression=expression, arg=arg)
- if get_main_world_prefix(frame.context) is None:
+ context = frame.context
+ # A page whose context is None (an engine's pre-attach/edge state) can't key
+ # the prefix WeakKeyDictionary — get_main_world_prefix(None) would raise
+ # instead of returning the no-prefix fallback, so take direct evaluate here.
+ if context is None or get_main_world_prefix(context) is None:
return await frame.evaluate(expression=expression, arg=arg)
if arg is not None and not _is_json_inlinable(arg):
return await frame.evaluate(expression=expression, arg=arg)
diff --git a/tests/unit/test_browser_object_identity_guard.py b/tests/unit/test_browser_object_identity_guard.py
new file mode 100644
index 000000000..57965a267
--- /dev/null
+++ b/tests/unit/test_browser_object_identity_guard.py
@@ -0,0 +1,296 @@
+"""AST guard against package-native browser-object identity checks in production.
+
+Skyvern selects among several browser engines whose ``Page`` / ``Frame`` /
+``Browser`` / ... classes have distinct identities. A runtime ``isinstance`` /
+``issubclass`` / ``type(x) is`` check against one driver's object class silently
+misclassifies an equivalent object from a different engine. This guard walks
+production ``skyvern/**`` and ``cloud/**`` and fails when a new such check
+appears; the structural predicates in ``skyvern.webeye.browser_object_predicates``
+are the sanctioned alternative.
+
+Deliberately out of scope (so this does not pretend the later exception-catch
+migration is finished, and does not raise false positives):
+ * annotations, ``TYPE_CHECKING`` imports, and ``cast(...)`` -- type-only usage,
+ never a runtime branch;
+ * ``except <native error>`` catches and identity checks against driver *error*
+ classes (Error/TimeoutError/TargetClosedError) -- that is a separate tranche;
+ * classes named Page/Frame/Response/... imported from non-driver packages
+ (fastapi, starlette, httpx, ...), resolved by import origin.
+"""
+
+from __future__ import annotations
+
+import ast
+from dataclasses import dataclass
+from pathlib import Path
+
+import pytest
+
+# Import roots of the engines whose object classes this guard polices. A new
+# engine adapter that ships its own Page/Frame-alike classes under a distinct
+# import root must be registered here, or identity checks against it slip past.
+_BROWSER_DRIVER_ROOTS = ("playwright", "patchright", "rustwright")
+
+# Browser *object* classes whose identity is engine-specific. Driver *error*
+# classes (Error, TimeoutError, TargetClosedError, ...) are intentionally absent:
+# migrating those catches is a separate tranche, not this guard's concern.
+_BROWSER_OBJECT_CLASSES = frozenset(
+ {
+ "Page",
+ "Frame",
+ "FrameLocator",
+ "Locator",
+ "ElementHandle",
+ "JSHandle",
+ "Browser",
+ "BrowserContext",
+ "BrowserType",
+ "CDPSession",
+ "Download",
+ "Request",
+ "Response",
+ "Route",
+ "WebSocket",
+ "WebSocketRoute",
+ "Worker",
+ "Dialog",
+ "ConsoleMessage",
+ "FileChooser",
+ "Video",
+ "Keyboard",
+ "Mouse",
+ "Touchscreen",
+ "Accessibility",
+ "Clock",
+ "Selectors",
+ "Tracing",
+ "Playwright",
+ }
+)
+
+
+@dataclass(frozen=True)
+class Violation:
+ filename: str
+ lineno: int
+ class_name: str
+ check: str
+
+
+def _module_root_is_driver(module: str | None) -> bool:
+ if not module:
+ return False
+ return module.split(".", 1)[0] in _BROWSER_DRIVER_ROOTS
+
+
+def _dotted_path(node: ast.expr) -> str | None:
+ parts: list[str] = []
+ while isinstance(node, ast.Attribute):
+ parts.append(node.attr)
+ node = node.value
+ if not isinstance(node, ast.Name):
+ return None
+ parts.append(node.id)
+ parts.reverse()
+ return ".".join(parts)
+
+
+class _IdentityCheckVisitor(ast.NodeVisitor):
+ def __init__(self, filename: str) -> None:
+ self.filename = filename
+ self.violations: list[Violation] = []
+ # local name -> originating driver object-class name (e.g. "P" -> "Frame")
+ self._driver_object_names: dict[str, str] = {}
+ # local module alias -> driver module (e.g. "pw" -> "playwright.async_api")
+ self._driver_module_aliases: dict[str, str] = {}
+
+ def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
+ if _module_root_is_driver(node.module):
+ for alias in node.names:
+ if alias.name in _BROWSER_OBJECT_CLASSES:
+ self._driver_object_names[alias.asname or alias.name] = alias.name
+ self.generic_visit(node)
+
+ def visit_Import(self, node: ast.Import) -> None:
+ for alias in node.names:
+ if _module_root_is_driver(alias.name):
+ self._driver_module_aliases[alias.asname or alias.name] = alias.name
+ self.generic_visit(node)
+
+ def _module_expr_is_driver(self, node: ast.expr) -> bool:
+ # ``module_alias.Page`` where the module part may be a single alias
+ # (``pw``) or a dotted package path (``playwright.async_api``). Match the
+ # rendered path against a registered driver import, resolving by origin
+ # (never by the ``Page`` text): an exact hit, or a submodule of an
+ # imported driver root (``import playwright`` + ``playwright.async_api.Page``).
+ path = _dotted_path(node)
+ if path is None:
+ return False
+ for alias in self._driver_module_aliases:
+ if path == alias or path.startswith(alias + "."):
+ return True
+ return False
+
+ def _resolve(self, node: ast.expr) -> str | None:
+ if isinstance(node, ast.Name):
+ return self._driver_object_names.get(node.id)
+ if isinstance(node, ast.Attribute) and node.attr in _BROWSER_OBJECT_CLASSES:
+ if self._module_expr_is_driver(node.value):
+ return node.attr
+ return None
+
+ def _record(self, node: ast.expr, check: str) -> None:
+ targets = node.elts if isinstance(node, ast.Tuple) else [node]
+ for target in targets:
+ class_name = self._resolve(target)
+ if class_name is not None:
+ self.violations.append(Violation(self.filename, target.lineno, class_name, check))
+
+ def visit_Call(self, node: ast.Call) -> None:
+ if isinstance(node.func, ast.Name) and node.func.id in ("isinstance", "issubclass") and len(node.args) >= 2:
+ self._record(node.args[1], node.func.id)
+ self.generic_visit(node)
+
+ def visit_Compare(self, node: ast.Compare) -> None:
+ left = node.left
+ is_type_call = isinstance(left, ast.Call) and isinstance(left.func, ast.Name) and left.func.id == "type"
+ if is_type_call:
+ for op, comparator in zip(node.ops, node.comparators):
+ if isinstance(op, (ast.Is, ast.IsNot, ast.Eq, ast.NotEq)):
+ self._record(comparator, "type-is")
+ self.generic_visit(node)
+
+
+def find_browser_object_identity_checks(source: str, filename: str = "<test>") -> list[Violation]:
+ visitor = _IdentityCheckVisitor(filename)
+ visitor.visit(ast.parse(source))
+ return visitor.violations
+
+
+# --- detector unit tests (synthetic sources) --------------------------------
+
+
+def test_flags_isinstance_against_driver_page() -> None:
+ src = "from playwright.async_api import Page\nx = isinstance(obj, Page)\n"
+ v = find_browser_object_identity_checks(src)
+ assert [(x.class_name, x.check) for x in v] == [("Page", "isinstance")]
+
+
+def test_flags_aliased_import() -> None:
+ src = "from patchright.async_api import Frame as F\nx = isinstance(obj, F)\n"
+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Frame"]
+
+
+def test_flags_each_member_of_isinstance_tuple() -> None:
+ src = "from playwright.async_api import Frame, Page\nx = isinstance(obj, (Page, Frame))\n"
+ assert sorted(x.class_name for x in find_browser_object_identity_checks(src)) == ["Frame", "Page"]
+
+
+def test_flags_issubclass_and_type_is() -> None:
+ src = "from rustwright import Browser\na = issubclass(cls, Browser)\nb = type(obj) is Browser\n"
+ checks = sorted(x.check for x in find_browser_object_identity_checks(src))
+ assert checks == ["issubclass", "type-is"]
+
+
+def test_flags_module_attribute_access() -> None:
+ src = "import playwright.async_api as pw\nx = isinstance(obj, pw.Page)\n"
+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Page"]
+
+
+def test_flags_unaliased_dotted_import_attribute_access() -> None:
+ src = "import playwright.async_api\nx = isinstance(obj, playwright.async_api.Page)\n"
+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Page"]
+
+
+def test_flags_dotted_attribute_access_under_bare_driver_root_import() -> None:
+ src = "import playwright\nx = isinstance(obj, playwright.async_api.Frame)\n"
+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Frame"]
+
+
+def test_flags_unaliased_dotted_import_for_other_driver_root() -> None:
+ src = "import patchright.async_api\nx = issubclass(cls, patchright.async_api.Browser)\n"
+ assert [x.class_name for x in find_browser_object_identity_checks(src)] == ["Browser"]
+
+
+def test_ignores_dotted_attribute_access_from_non_driver_package() -> None:
+ src = "import fastapi.responses\nx = isinstance(summary, fastapi.responses.Response)\n"
+ assert find_browser_object_identity_checks(src) == []
+
+
+def test_ignores_same_named_class_from_non_driver_package() -> None:
+ src = "from fastapi import Response\nx = isinstance(summary, Response)\n"
+ assert find_browser_object_identity_checks(src) == []
+
+
+def test_ignores_driver_error_classes() -> None:
+ src = "from playwright.async_api import TimeoutError as PWTimeout\nx = isinstance(exc, PWTimeout)\n"
+ assert find_browser_object_identity_checks(src) == []
+
+
+def test_ignores_annotations_and_cast() -> None:
+ src = (
+ "from typing import cast\n"
+ "from playwright.async_api import Page\n"
+ "def f(x: Page) -> Page:\n"
+ " y: Page = x\n"
+ " return cast(Page, y)\n"
+ )
+ assert find_browser_object_identity_checks(src) == []
+
+
+def test_ignores_type_checking_import_used_only_in_annotation() -> None:
+ src = (
+ "from typing import TYPE_CHECKING\n"
+ "if TYPE_CHECKING:\n"
+ " from playwright.async_api import Page\n"
+ "def f(x: 'Page') -> None:\n"
+ " return None\n"
+ )
+ assert find_browser_object_identity_checks(src) == []
+
+
+# --- repo enforcement -------------------------------------------------------
+
+
+def _production_python_files() -> list[Path]:
+ repo_root = Path(__file__).resolve().parents[2]
+ files: list[Path] = []
+ for package in ("skyvern", "cloud"):
+ root = repo_root / package
+ if not root.is_dir():
+ continue
+ for path in root.rglob("*.py"):
+ parts = set(path.parts)
+ if "tests" in parts or path.name.startswith("test_") or "__pycache__" in parts:
+ continue
+ files.append(path)
+ return files
+
+
+def test_no_browser_object_identity_checks_in_production() -> None:
+ offenders: list[Violation] = []
+ for path in _production_python_files():
+ try:
+ tree_src = path.read_text(encoding="utf-8")
+ except (OSError, UnicodeDecodeError):
+ continue
+ try:
+ offenders.extend(find_browser_object_identity_checks(tree_src, str(path)))
+ except SyntaxError:
+ continue
+ assert not offenders, (
+ "Package-native browser-object identity checks found. Use the structural "
+ "predicates in skyvern.webeye.browser_object_predicates (e.g. is_page_like) "
+ "instead of isinstance/issubclass/type() against a driver's object class:\n"
+ + "\n".join(f" {o.filename}:{o.lineno} {o.check}({o.class_name})" for o in offenders)
+ )
+
+
+def test_repo_scan_finds_at_least_the_production_tree() -> None:
+ # Guard against the enforcement test silently passing because the scan walked
+ # nothing (wrong root, glob typo). skyvern/** must always yield files.
+ assert len(_production_python_files()) > 100
+
+
+if __name__ == "__main__":
+ raise SystemExit(pytest.main([__file__, "-v"]))
diff --git a/tests/unit/webeye/test_browser_object_predicates.py b/tests/unit/webeye/test_browser_object_predicates.py
new file mode 100644
index 000000000..a70496277
--- /dev/null
+++ b/tests/unit/webeye/test_browser_object_predicates.py
@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from playwright.async_api import Frame, Page
+
+from skyvern.webeye.browser_object_predicates import is_page_like
+
+
+def test_real_page_class_instance_is_page_like() -> None:
+ page = MagicMock(spec=Page)
+ assert is_page_like(page) is True
+
+
+def test_real_frame_class_instance_is_not_page_like() -> None:
+ frame = MagicMock(spec=Frame)
+ assert is_page_like(frame) is False
+
+
+def test_unrelated_object_is_not_page_like() -> None:
+ assert is_page_like(object()) is False
+ assert is_page_like(ValueError("boom")) is False
+ assert is_page_like(None) is False
+
+
+def test_matches_isinstance_page_across_representative_objects() -> None:
+ # The predicate must be a drop-in for isinstance(obj, Page) at the one
+ # migrated call site: identical verdict for a Page, a Frame, and their
+ # spec'd mocks (which back the existing suite).
+ for obj in (MagicMock(spec=Page), MagicMock(spec=Frame), object(), "not-a-page"):
+ assert is_page_like(obj) == isinstance(obj, Page)
+
+
+def test_page_like_requires_all_page_only_capabilities() -> None:
+ # A frame-tree owner exposes main_frame + bring_to_front; missing either
+ # marker is not enough to be a page.
+ only_main_frame = MagicMock(spec=["main_frame", "url", "evaluate"])
+ assert is_page_like(only_main_frame) is False
+
+
+def test_page_like_rejects_object_carrying_frame_ownership_markers() -> None:
+ # Even a duck object that exposes every page capability is rejected when it
+ # also reports a parent page/frame -- that is a subframe, not a page.
+ frame_shaped = MagicMock(spec=["main_frame", "context", "bring_to_front", "evaluate", "page", "parent_frame"])
+ assert is_page_like(frame_shaped) is False
+
+
+def test_page_like_rejects_object_missing_context() -> None:
+ # context backs the PageLike TypeGuard (the call site dereferences
+ # frame.context), so an object without it must not narrow to PageLike even
+ # with both page-vs-frame discriminators present.
+ no_context = MagicMock(spec=["main_frame", "bring_to_front", "evaluate"])
+ assert is_page_like(no_context) is False
+
+
+def test_page_like_rejects_object_missing_evaluate() -> None:
+ # evaluate is a PageLike member dereferenced on the narrowed page; missing it
+ # means the object does not fully satisfy the contract.
+ no_evaluate = MagicMock(spec=["main_frame", "context", "bring_to_front"])
+ assert is_page_like(no_evaluate) is False
+
+
+def test_engine_neutral_page_from_a_distinct_class_is_page_like() -> None:
+ # A second engine's Page has a distinct class identity, so isinstance against
+ # a concrete driver's Page would return False for it; the structural
+ # predicate accepts it because it genuinely presents the full PageLike
+ # capability surface (main_frame, context, bring_to_front, evaluate).
+ class OtherEnginePage:
+ main_frame = object()
+ context = object()
+
+ def bring_to_front(self) -> None: ...
+
+ def evaluate(self, expression: str, arg: object = None) -> None: ...
+
+ assert isinstance(OtherEnginePage(), Page) is False
+ assert is_page_like(OtherEnginePage()) is True
diff --git a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py b/tests/unit/webeye/test_skyvern_frame_main_world_routing.py
index 4dee9c234..719f6b346 100644
--- a/tests/unit/webeye/test_skyvern_frame_main_world_routing.py
+++ b/tests/unit/webeye/test_skyvern_frame_main_world_routing.py
@@ -94,6 +94,22 @@ class TestSkyvernFrameEvaluateRouting:
page.evaluate.assert_awaited_once_with(expression="(el) => el.blur()", arg=element_handle_like)
page.context.new_cdp_session.assert_not_awaited() # type: ignore[attr-defined]
+ @pytest.mark.asyncio
+ async def test_page_like_with_none_context_falls_back_to_direct_evaluate(self) -> None:
+ """A page-like object whose context is None (a future engine's pre-attach
+ state) must take direct frame.evaluate and never reach the prefix
+ WeakKeyDictionary, whose get(None) would raise TypeError."""
+ page = MagicMock(spec=Page)
+ page.context = None
+ page.evaluate = AsyncMock(return_value="page-evaluate-result")
+
+ with patch("skyvern.webeye.utils.page.get_main_world_prefix") as get_prefix:
+ result = await SkyvernFrame.evaluate(page, "() => 1")
+
+ assert result == "page-evaluate-result"
+ page.evaluate.assert_awaited_once_with(expression="() => 1", arg=None)
+ get_prefix.assert_not_called()
+
@pytest.mark.asyncio
async def test_frame_target_never_routes_through_runtime_evaluate(self) -> None:
"""Iframe Frames have their own world; page-level Runtime.evaluate lands elsewhere."""