Skip to content

Automate browser based workflows with AI

Block internal navigation hops before the browser follows them (SKY-13377) (#7992)

Aaron Perez7f72c33b84f7
diff --git a/skyvern/core/script_generations/script_skyvern_page.py b/skyvern/core/script_generations/script_skyvern_page.py
index cae32d741..ab5e810c1 100644
--- a/skyvern/core/script_generations/script_skyvern_page.py
+++ b/skyvern/core/script_generations/script_skyvern_page.py
@@ -69,6 +69,7 @@ from skyvern.webeye.actions.handler import (
from skyvern.webeye.actions.responses import ActionFailure, ActionResult, ActionSuccess
from skyvern.webeye.browser_engine import BrowserEngineSelection
from skyvern.webeye.browser_state import BrowserState
+from skyvern.webeye.navigation_egress_guard import install_navigation_egress_guard
from skyvern.webeye.scraper.scraped_page import ScrapedPage
from skyvern.webeye.utils.page import SkyvernFrame
@@ -1051,6 +1052,11 @@ class ScriptSkyvernPage(SkyvernPage):
timeout = kwargs.pop("timeout", settings.BROWSER_LOADING_TIMEOUT_MS)
max_retries = kwargs.pop("max_retries", NAVIGATION_MAX_RETRY_TIME)
+ # This flow acquires the browser with navigate=False and drives page.goto itself, so the
+ # context page event is the only other thing that would arm the guard — and it can lose the
+ # race against a goto issued in the same tick.
+ await install_navigation_egress_guard(self.page)
+
# Retry logic matching agent mode (real_browser_state.navigate_to_url)
last_error: Exception | None = None
for attempt in range(max_retries):
diff --git a/skyvern/webeye/browser_factory.py b/skyvern/webeye/browser_factory.py
index e59fb38d4..935fca1a2 100644
--- a/skyvern/webeye/browser_factory.py
+++ b/skyvern/webeye/browser_factory.py
@@ -59,6 +59,7 @@ from skyvern.webeye.cdp_connection import (
)
from skyvern.webeye.cdp_download_interceptor import CDPDownloadInterceptor, bind_download_interceptor_to_context
from skyvern.webeye.dialog_handler import set_dialog_handler
+from skyvern.webeye.navigation_egress_guard import arm_navigation_egress_guard
from skyvern.webeye.session_cookies import restore_banked_cookies, restore_session_cookies
LOG = structlog.get_logger()
@@ -709,6 +710,7 @@ class BrowserContextFactory:
headers=scoped_headers,
route_handlers_allowed=route_handlers_allowed,
)
+ arm_navigation_egress_guard(browser_context)
proxy_location: ProxyLocationInput = kwargs.get("proxy_location")
if isinstance(proxy_location, ProxyLocation):
diff --git a/skyvern/webeye/navigation_egress_guard.py b/skyvern/webeye/navigation_egress_guard.py
new file mode 100644
index 000000000..1b98c902f
--- /dev/null
+++ b/skyvern/webeye/navigation_egress_guard.py
@@ -0,0 +1,150 @@
+"""Classify every navigation hop before the browser connects to it, over the CDP Fetch domain.
+
+``navigate_with_retry`` validates the destination before dispatch and revalidates the followed
+chain after ``page.goto`` returns, but a redirect the browser already followed cannot be undone
+by then: the internal request was issued and its response scripts may have run. This guard closes
+that window by classifying each hop while it is paused.
+
+Why CDP ``Fetch`` and not Playwright routing (probed on Chromium 128, playwright 1.4x):
+``context.route("**/*")`` does **not** re-fire on a 3xx target (playwright#3993/#34994) — a handler
+on a ``/entry -> /hop1 -> /final`` chain sees only ``/entry`` while the browser follows the rest.
+Reaching the hops through routing therefore requires re-issuing every request via
+``route.fetch(max_redirects=0)``, which serves the response from Playwright's own network stack:
+that drops all four ``Sec-Fetch-*`` metadata headers and reorders the rest, and (because
+``route.fulfill`` terminates the chain) starves the co-registered header route in
+``browser_factory._apply_origin_scoped_headers``. ``Fetch.requestPaused`` re-fires on every hop
+with the browser doing the fetching, so neither cost applies.
+
+Scope and limits:
+- Pauses ``Document`` requests only, so subresources are untouched; sub-frame navigations are
+ ``Document`` requests and are covered.
+- Classification is ``validate_navigation_destination`` — deliberately the same function used
+ before dispatch and after return, so the three checks cannot drift apart.
+- That classifier does not resolve DNS, so a public hostname with an internal A record still
+ passes, and a rebinding host can flip between check and connect regardless. Both need
+ enforcement below the browser (network policy); no userspace guard closes them.
+- Installation is best-effort: a browser that refuses a CDP session keeps only the pre-dispatch
+ and post-return checks rather than losing navigation entirely.
+- **Popups are NOT covered.** Playwright exposes a popup ``Page`` only after its initial
+ navigation has begun, so ``window.open`` / ``target=_blank`` straight to an internal URL wins
+ the race against the install (probed: the internal body reached the popup). The only userspace
+ fix is a Playwright ``context.route``, which closes it — but registering any route makes
+ Chromium disable its HTTP cache and add ``Pragma``/``Cache-Control: no-cache`` to every
+ request, and routes are already disallowed on some runs, so it is neither free nor universal.
+ Popup egress belongs to below-browser network policy, alongside DNS rebinding.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import structlog
+from playwright.async_api import BrowserContext, CDPSession, Page
+
+from skyvern.exceptions import BlockedNavigationDestination
+from skyvern.webeye.navigation import validate_navigation_destination
+
+LOG = structlog.get_logger()
+
+_GUARD_ATTR = "_skyvern_navigation_egress_guard"
+
+# requestStage Request pauses the hop before it leaves the browser; resourceType Document keeps
+# the pause count at roughly one per navigation instead of one per subresource.
+_FETCH_PATTERNS: list[dict[str, str]] = [{"urlPattern": "*", "resourceType": "Document", "requestStage": "Request"}]
+
+
+class NavigationEgressGuard:
+ def __init__(self, cdp_session: CDPSession) -> None:
+ self._cdp_session = cdp_session
+ self._tasks: set[asyncio.Task[None]] = set()
+
+ def on_request_paused(self, event: dict[str, Any]) -> None:
+ task = asyncio.create_task(self._decide(event))
+ self._tasks.add(task)
+ task.add_done_callback(self._tasks.discard)
+
+ async def _decide(self, event: dict[str, Any]) -> None:
+ request_id = event.get("requestId")
+ if not request_id:
+ return
+ url = (event.get("request") or {}).get("url") or ""
+
+ try:
+ validate_navigation_destination(url)
+ except BlockedNavigationDestination as error:
+ LOG.warning("Blocked navigation hop to an internal destination", url=url, reason=error.reason)
+ await self._answer("Fetch.failRequest", {"requestId": request_id, "errorReason": "AddressUnreachable"})
+ return
+ except Exception:
+ # A paused request that never receives a verdict hangs the navigation until its
+ # timeout, so an unexpected classifier failure still has to answer — and blocking
+ # is the only answer that cannot leak an internal response.
+ LOG.exception("Navigation egress classification failed; blocking the hop", url=url)
+ await self._answer("Fetch.failRequest", {"requestId": request_id, "errorReason": "AddressUnreachable"})
+ return
+
+ await self._answer("Fetch.continueRequest", {"requestId": request_id})
+
+ async def _answer(self, method: str, params: dict[str, Any]) -> None:
+ try:
+ await self._cdp_session.send(method, params)
+ except Exception:
+ # The session detaches when the page closes and the pause dies with it; a lost verdict
+ # on an already-gone request is expected, not a failure to enforce.
+ LOG.debug("Navigation egress verdict could not be delivered", method=method, exc_info=True)
+
+
+async def install_navigation_egress_guard(page: Page) -> None:
+ """Attach the per-hop guard to ``page`` once; subsequent calls are no-ops."""
+ if getattr(page, _GUARD_ATTR, None) is not None:
+ return
+
+ try:
+ cdp_session = await page.context.new_cdp_session(page)
+ guard = NavigationEgressGuard(cdp_session)
+ cdp_session.on("Fetch.requestPaused", guard.on_request_paused)
+ await cdp_session.send("Fetch.enable", {"patterns": _FETCH_PATTERNS})
+ except Exception:
+ LOG.warning(
+ "Could not install the per-hop navigation egress guard; "
+ "navigation keeps only pre-dispatch and post-return validation",
+ exc_info=True,
+ )
+ return
+
+ page._skyvern_navigation_egress_guard = guard # type: ignore[attr-defined]
+
+
+def arm_navigation_egress_guard(browser_context: BrowserContext) -> None:
+ """Arm every page of ``browser_context``, including ones the site opens itself.
+
+ Binding the guard to a navigation helper leaves anything that does not call one unguarded:
+ popups, ``target=_blank``, and generated scripts, which acquire the browser with
+ ``navigate=False`` and then drive ``page.goto`` directly. Binding it to the context does not.
+
+ The page event schedules the install, so a page that navigates in the same tick can still
+ outrun it; the navigation entry points also await ``install_navigation_egress_guard``, which
+ is idempotent, to close that race where a caller exists.
+
+ Best-effort by construction: this runs inside browser-context creation, where any raise
+ becomes UnknownErrorWhileCreatingBrowserContext and fails the whole launch. A guard layered
+ on top of two existing validators must never be able to do that.
+ """
+ pending: set[asyncio.Task[None]] = set()
+
+ def on_page(page: Page) -> None:
+ task = asyncio.create_task(install_navigation_egress_guard(page))
+ pending.add(task)
+ task.add_done_callback(pending.discard)
+
+ try:
+ browser_context.on("page", on_page)
+ for page in browser_context.pages:
+ on_page(page)
+ except Exception:
+ LOG.warning(
+ "Could not arm the navigation egress guard on the browser context; "
+ "navigation entry points still install it per page",
+ exc_info=True,
+ )
diff --git a/skyvern/webeye/real_browser_state.py b/skyvern/webeye/real_browser_state.py
index bf7ce380f..d5094fdbc 100644
--- a/skyvern/webeye/real_browser_state.py
+++ b/skyvern/webeye/real_browser_state.py
@@ -33,6 +33,7 @@ from skyvern.webeye.browser_factory import BrowserCleanupFunc, BrowserContextFac
from skyvern.webeye.browser_state import BrowserState
from skyvern.webeye.cdp_download_interceptor import disable_download_interceptor_for_context
from skyvern.webeye.navigation import is_permanent_navigation_error, navigate_with_retry
+from skyvern.webeye.navigation_egress_guard import install_navigation_egress_guard
from skyvern.webeye.scraper import scraper
from skyvern.webeye.scraper.scraped_page import CleanupElementTreeFunc, ScrapedPage, ScrapeExcludeFunc
from skyvern.webeye.session_cookies import persist_session_cookies
@@ -262,6 +263,9 @@ class RealBrowserState(BrowserState):
retry_times: int = NAVIGATION_MAX_RETRY_TIME,
wait_until: Literal["load", "domcontentloaded", "commit"] = "load",
) -> None:
+ # Stays attached for the page's lifetime, so later same-page navigations (link clicks,
+ # script redirects, form posts) are classified per hop too, not just this goto.
+ await install_navigation_egress_guard(page)
await navigate_with_retry(
navigate=lambda strategy: page.goto(url, timeout=settings.BROWSER_LOADING_TIMEOUT_MS, wait_until=strategy),
url=url,
diff --git a/tests/unit_tests/webeye/test_navigation_egress_guard.py b/tests/unit_tests/webeye/test_navigation_egress_guard.py
new file mode 100644
index 000000000..de73581b5
--- /dev/null
+++ b/tests/unit_tests/webeye/test_navigation_egress_guard.py
@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+import pytest
+
+from skyvern.webeye.navigation_egress_guard import (
+ NavigationEgressGuard,
+ install_navigation_egress_guard,
+)
+
+
+class _FakeCDPSession:
+ def __init__(self, *, fail_on: str | None = None) -> None:
+ self.sent: list[tuple[str, dict[str, Any]]] = []
+ self.listeners: dict[str, Any] = {}
+ self._fail_on = fail_on
+
+ def on(self, event: str, handler: Any) -> None:
+ self.listeners[event] = handler
+
+ async def send(self, method: str, params: dict[str, Any] | None = None) -> None:
+ if self._fail_on == method:
+ raise RuntimeError("cdp session detached")
+ self.sent.append((method, params or {}))
+
+ def verdicts(self) -> list[tuple[str, str]]:
+ return [
+ (method, params.get("requestId", ""))
+ for method, params in self.sent
+ if method.startswith("Fetch.") and method != "Fetch.enable"
+ ]
+
+
+class _FakeContext:
+ def __init__(self, session: _FakeCDPSession | None, *, raises: bool = False) -> None:
+ self._session = session
+ self._raises = raises
+
+ async def new_cdp_session(self, page: Any) -> _FakeCDPSession:
+ if self._raises or self._session is None:
+ raise RuntimeError("cdp sessions unsupported")
+ return self._session
+
+
+class _FakePage:
+ def __init__(self, session: _FakeCDPSession | None, *, raises: bool = False) -> None:
+ self.context = _FakeContext(session, raises=raises)
+
+
+def _paused(url: str, request_id: str = "req-1") -> dict[str, Any]:
+ return {"requestId": request_id, "resourceType": "Document", "request": {"url": url}}
+
+
+async def _decide(session: _FakeCDPSession, event: dict[str, Any]) -> None:
+ guard = NavigationEgressGuard(session)
+ guard.on_request_paused(event)
+ # on_request_paused schedules the verdict; let the scheduled task run.
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+
+
+BLOCKED_HOPS = [
+ pytest.param("http://169.254.169.254/latest/meta-data/", id="cloud-metadata-ip"),
+ pytest.param("http://127.0.0.1:9000/", id="loopback-ip"),
+ pytest.param("http://10.0.0.5/", id="private-10"),
+ pytest.param("http://192.168.0.10/admin", id="private-192"),
+ pytest.param("http://localhost:8000/admin", id="localhost"),
+ pytest.param("http://metadata.google.internal/computeMetadata/v1/", id="metadata-hostname"),
+ pytest.param("http://kubernetes.default.svc/api", id="cluster-internal-svc"),
+ pytest.param("http://2130706433/", id="decimal-ip-loopback"),
+ pytest.param("http://0xa9fea9fe/", id="hex-ip-metadata"),
+ pytest.param("file:///etc/passwd", id="local-file"),
+]
+
+
[email protected]("url", BLOCKED_HOPS)
+async def test_internal_hop_is_failed_before_the_browser_connects(url: str) -> None:
+ session = _FakeCDPSession()
+ await _decide(session, _paused(url))
+ assert session.verdicts() == [("Fetch.failRequest", "req-1")]
+ assert session.sent[0][1]["errorReason"] == "AddressUnreachable"
+
+
[email protected]("url", ["https://example.com/", "http://93.184.216.34/", "https://sub.example.co.uk/a?b=c"])
+async def test_public_hop_continues(url: str) -> None:
+ session = _FakeCDPSession()
+ await _decide(session, _paused(url))
+ assert session.verdicts() == [("Fetch.continueRequest", "req-1")]
+
+
+async def test_unexpected_classifier_failure_blocks_rather_than_hangs(monkeypatch: pytest.MonkeyPatch) -> None:
+ def explode(url: str) -> None:
+ raise ValueError("classifier blew up")
+
+ monkeypatch.setattr("skyvern.webeye.navigation_egress_guard.validate_navigation_destination", explode)
+ session = _FakeCDPSession()
+ await _decide(session, _paused("https://example.com/"))
+ # A paused hop with no verdict hangs the navigation, so the only safe answer is to block.
+ assert session.verdicts() == [("Fetch.failRequest", "req-1")]
+
+
+async def test_detached_session_does_not_raise_out_of_the_handler() -> None:
+ session = _FakeCDPSession(fail_on="Fetch.failRequest")
+ await _decide(session, _paused("http://169.254.169.254/"))
+ assert session.verdicts() == []
+
+
+async def test_install_enables_document_stage_interception() -> None:
+ session = _FakeCDPSession()
+ page = _FakePage(session)
+ await install_navigation_egress_guard(page)
+
+ assert "Fetch.requestPaused" in session.listeners
+ patterns = dict(session.sent)["Fetch.enable"]["patterns"]
+ # Request stage is what makes the pause happen *before* the hop leaves the browser, and
+ # Document scoping is what keeps subresources out of the guard. Both are load-bearing.
+ assert patterns == [{"urlPattern": "*", "resourceType": "Document", "requestStage": "Request"}]
+
+
+async def test_install_is_idempotent_per_page() -> None:
+ session = _FakeCDPSession()
+ page = _FakePage(session)
+ await install_navigation_egress_guard(page)
+ await install_navigation_egress_guard(page)
+ assert [method for method, _ in session.sent].count("Fetch.enable") == 1
+
+
+async def test_install_failure_leaves_navigation_usable() -> None:
+ page = _FakePage(None, raises=True)
+ await install_navigation_egress_guard(page) # must not raise
+
+
+class _FakeBrowserContext:
+ """Duck-types the slice of BrowserContext that context-level arming uses."""
+
+ def __init__(self, pages: list[Any] | None = None) -> None:
+ self.pages = pages or []
+ self._handlers: dict[str, Any] = {}
+
+ def on(self, event: str, handler: Any) -> None:
+ self._handlers[event] = handler
+
+ def emit_page(self, page: Any) -> None:
+ self._handlers["page"](page)
+
+
+async def test_arming_the_context_covers_pages_that_never_call_a_navigation_helper() -> None:
+ """Popups, target=_blank, and generated scripts never reach a navigation helper.
+
+ Binding the guard only to a call site leaves every one of them unguarded.
+ """
+ from skyvern.webeye.navigation_egress_guard import arm_navigation_egress_guard
+
+ existing_session = _FakeCDPSession()
+ popup_session = _FakeCDPSession()
+ context = _FakeBrowserContext(pages=[_FakePage(existing_session)])
+
+ arm_navigation_egress_guard(context)
+ context.emit_page(_FakePage(popup_session))
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+
+ assert "Fetch.enable" in dict(existing_session.sent), "page open at arm time was left unguarded"
+ assert "Fetch.enable" in dict(popup_session.sent), "page opened by the site was left unguarded"
+
+
+async def test_arming_never_breaks_browser_context_creation() -> None:
+ """Arming runs inside create_browser_context, where a raise fails the whole browser launch.
+
+ A guard layered on top of two existing validators must degrade, never take the browser down.
+ """
+ from skyvern.webeye.navigation_egress_guard import arm_navigation_egress_guard
+
+ class _ContextWithoutEvents:
+ pages: list[Any] = []
+
+ arm_navigation_egress_guard(_ContextWithoutEvents()) # must not raise
diff --git a/tests/unit_tests/webeye/test_navigation_retry.py b/tests/unit_tests/webeye/test_navigation_retry.py
index 1014921a7..65e39d471 100644
--- a/tests/unit_tests/webeye/test_navigation_retry.py
+++ b/tests/unit_tests/webeye/test_navigation_retry.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
@@ -367,3 +368,57 @@ async def test_navigate_with_retry_revalidates_every_redirect_hop() -> None:
)
settle.assert_not_awaited()
+
+
+async def test_navigate_to_url_installs_the_per_hop_egress_guard(monkeypatch: pytest.MonkeyPatch) -> None:
+ """The choke point must arm per-hop enforcement, not just pre/post validation.
+
+ Without this, dropping the install call leaves both validators green while every redirect
+ hop goes unclassified again — the exact gap the guard exists to close.
+ """
+ guarded: list[object] = []
+
+ async def record(page: object) -> None:
+ guarded.append(page)
+
+ monkeypatch.setattr("skyvern.webeye.real_browser_state.install_navigation_egress_guard", record)
+
+ page = AsyncMock()
+ page.goto = AsyncMock(return_value=None)
+ browser_state = RealBrowserState(pw=AsyncMock())
+ browser_state._wait_for_settle = AsyncMock()
+ browser_state._wait_for_challenge_solver = AsyncMock()
+
+ await browser_state.navigate_to_url(page=page, url="https://example.com/")
+
+ assert guarded == [page]
+
+
+async def test_generated_script_goto_arms_the_egress_guard(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Generated scripts acquire the browser with navigate=False and drive page.goto themselves.
+
+ That path reaches neither navigate_with_retry nor its pre-dispatch validation, so it has to
+ arm the per-hop guard itself or a redirect to an internal host goes unclassified.
+ """
+ # script_skyvern_page has a pre-existing import cycle with script_service (reproduces on
+ # main without this test); importing the service first establishes a working order.
+ import skyvern.services.script_service # noqa: F401
+ from skyvern.core.script_generations.script_skyvern_page import ScriptSkyvernPage
+
+ guarded: list[object] = []
+
+ async def record(page: object) -> None:
+ guarded.append(page)
+
+ monkeypatch.setattr("skyvern.core.script_generations.script_skyvern_page.install_navigation_egress_guard", record)
+
+ page = AsyncMock()
+ page.goto = AsyncMock(return_value=None)
+ # SkyvernPage subclasses playwright's Page, so a real instance needs a live connection;
+ # goto only reads self.page, so drive it with a stand-in self.
+ await ScriptSkyvernPage.goto(SimpleNamespace(page=page), "https://example.com/")
+
+ assert guarded == [page]
+ page.goto.assert_awaited_once()