Skip to content

Automate browser based workflows with AI

[SKY-12007] Add engine-neutral browser error taxonomy (#7468)

LawyZheng6b0b4fdc77d2
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 2031e1ff6..6d6999917 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -136,13 +136,22 @@ def _is_browser_connection_error(message: str) -> bool:
# A raw CDP connect failure (e.g. from playwright.chromium.connect_over_cdp) echoes the
-# ws/wss endpoint URL, which can carry the remote-browser vendor host, a session-bearing
-# path/query, or credentials embedded as user:pass@host. Redact it before it reaches a user.
-_CDP_WS_URL_RE = re.compile(r"wss?://\S+", re.IGNORECASE)
+# endpoint URL, which can carry the remote-browser vendor host, a session-bearing path/query,
+# or credentials embedded as user:pass@host. The devtools socket is always ws/wss, so a ws/wss
+# URL in a browser error is unambiguously a CDP endpoint and safe to redact anywhere. The
+# /json/version discovery endpoint is http/https and carries the same host/token, but so does an
+# ordinary navigation/target/proxy URL — an http(s) URL is only known to be a CDP endpoint in a
+# CDP-connection context, so http(s) redaction is scoped to that context (see redact_cdp_endpoint_urls).
+_WS_ENDPOINT_URL_RE = re.compile(r"wss?://\S+", re.IGNORECASE)
+_CDP_ENDPOINT_URL_RE = re.compile(r"(?:wss?|https?)://\S+", re.IGNORECASE)
-def _redact_cdp_endpoint_urls(message: str) -> str:
- return _CDP_WS_URL_RE.sub("[remote browser endpoint]", message)
+def redact_ws_endpoint_urls(message: str) -> str:
+ return _WS_ENDPOINT_URL_RE.sub("[remote browser endpoint]", message)
+
+
+def redact_cdp_endpoint_urls(message: str) -> str:
+ return _CDP_ENDPOINT_URL_RE.sub("[remote browser endpoint]", message)
def get_user_facing_exception_message(exception: Exception) -> str:
@@ -506,7 +515,7 @@ class UnknownErrorWhileCreatingBrowserContext(SkyvernException):
if isinstance(exception, CdpConnectionConfigurationError):
return exception.message or str(exception)
- raw_message = _redact_cdp_endpoint_urls(str(exception).strip())
+ raw_message = redact_cdp_endpoint_urls(str(exception).strip())
raw_lower = raw_message.lower()
# Browser launch environment errors: worker cannot initialize the
diff --git a/skyvern/webeye/browser_errors.py b/skyvern/webeye/browser_errors.py
new file mode 100644
index 000000000..e77295a16
--- /dev/null
+++ b/skyvern/webeye/browser_errors.py
@@ -0,0 +1,134 @@
+"""Engine-neutral browser-error taxonomy and classifier.
+
+Skyvern can select among several browser engines (for example stock Playwright or a
+private engine), each of which raises its own native error classes with distinct
+identities. This module owns a taxonomy that recovery code can branch on without
+importing any engine, plus a classifier parameterized by the *selected* engine's
+native error families. It deliberately imports no browser driver at module scope so
+it stays driver-agnostic and reusable by every adapter.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from skyvern.exceptions import SkyvernException, redact_cdp_endpoint_urls, redact_ws_endpoint_urls
+
+ExceptionType = type[BaseException]
+ErrorPredicate = Callable[[BaseException], bool]
+
+
+class BrowserAutomationError(SkyvernException):
+ """Engine-neutral base for a browser failure classified from a driver-native error."""
+
+
+class BrowserTimeoutError(BrowserAutomationError):
+ """A driver operation exceeded its deadline."""
+
+
+class BrowserTargetClosedError(BrowserAutomationError):
+ """The target page, frame, context, or browser was closed underneath an operation."""
+
+
+class BrowserCdpConnectionError(BrowserAutomationError):
+ """A CDP/transport-level failure talking to the browser."""
+
+
+class BrowserRetryableCdpError(BrowserCdpConnectionError):
+ """A CDP/transport failure the caller may retry (e.g. a transient disconnect)."""
+
+
+class BrowserErrorFamiliesConfigError(ValueError):
+ """Raised when an engine's error-family configuration is invalid or ambiguous."""
+
+
+_FAMILY_TYPE_FIELDS = (
+ "timeout_types",
+ "target_closed_types",
+ "cdp_connection_types",
+ "retryable_types",
+ "base_error_types",
+)
+
+
+@dataclass(frozen=True)
+class BrowserEngineErrorFamilies:
+ """Native error classes/predicates for one selected engine, grouped by taxonomy; a native type
+ appearing in more than one family is rejected at construction time (rather than resolved by
+ isinstance ordering) to keep classification unambiguous, and the ``cdp_connection_predicate`` /
+ ``retryable_predicate`` run on *every* exception that reaches their step unmatched by an earlier
+ family's types, so a predicate must scope itself by the engine's identity (type-check, or match
+ transport signals only that engine emits) — one keyed solely on message substrings will also
+ claim unrelated exceptions carrying the same text.
+ """
+
+ timeout_types: tuple[ExceptionType, ...] = ()
+ target_closed_types: tuple[ExceptionType, ...] = ()
+ cdp_connection_types: tuple[ExceptionType, ...] = ()
+ retryable_types: tuple[ExceptionType, ...] = ()
+ base_error_types: tuple[ExceptionType, ...] = ()
+ cdp_connection_predicate: ErrorPredicate | None = None
+ retryable_predicate: ErrorPredicate | None = None
+
+ def __post_init__(self) -> None:
+ seen: dict[ExceptionType, str] = {}
+ for field_name in _FAMILY_TYPE_FIELDS:
+ for entry in getattr(self, field_name):
+ if not (isinstance(entry, type) and issubclass(entry, BaseException)):
+ raise BrowserErrorFamiliesConfigError(
+ f"{field_name} entry {entry!r} is not a BaseException subclass"
+ )
+ prior = seen.get(entry)
+ if prior is not None:
+ location = f"twice in {field_name}" if prior == field_name else f"in both {prior} and {field_name}"
+ raise BrowserErrorFamiliesConfigError(
+ f"{entry.__name__} appears {location}; a native error type may only belong to one family"
+ )
+ seen[entry] = field_name
+
+
+def _matches(exc: BaseException, types: tuple[ExceptionType, ...], predicate: ErrorPredicate | None) -> bool:
+ if types and isinstance(exc, types):
+ return True
+ return predicate is not None and predicate(exc)
+
+
+def _wrap(cls: type[BrowserAutomationError], exc: BaseException) -> BrowserAutomationError:
+ """Wrap a native error as ``cls``, redacting browser-endpoint URLs from ``.message`` while keeping
+ the raw native error on ``__cause__`` for logs and diagnostics. A ws/wss devtools socket URL is
+ unambiguously a CDP endpoint, so it is redacted for every family; an http/https URL is only known
+ to be a CDP discovery endpoint (rather than an ordinary navigation/target/proxy URL the user wants
+ to see) when ``cls`` is a CDP-connection family, so http(s) redaction is scoped there.
+ """
+ raw = str(exc)
+ redacted = (
+ redact_cdp_endpoint_urls(raw) if issubclass(cls, BrowserCdpConnectionError) else redact_ws_endpoint_urls(raw)
+ )
+ error = cls(redacted)
+ error.__cause__ = exc
+ return error
+
+
+def classify_browser_error(
+ exc: BaseException,
+ families: BrowserEngineErrorFamilies,
+) -> BrowserAutomationError | None:
+ """Map a native engine error to the taxonomy, or return ``None`` if unrecognized. Precedence is
+ most-specific first — retryable CDP, target closed, timeout, CDP transport, then the engine's
+ generic base error — so a type matching several families via inheritance (e.g. a timeout that
+ subclasses the base error) resolves to the earliest match; ``None`` means the error is not in
+ this engine's taxonomy and the caller must re-raise it (never swallowed or flattened into the
+ base type), and positive classifications preserve the original native exception as ``__cause__``.
+ """
+ if _matches(exc, families.retryable_types, families.retryable_predicate):
+ return _wrap(BrowserRetryableCdpError, exc)
+ if _matches(exc, families.target_closed_types, None):
+ return _wrap(BrowserTargetClosedError, exc)
+ if _matches(exc, families.timeout_types, None):
+ return _wrap(BrowserTimeoutError, exc)
+ if _matches(exc, families.cdp_connection_types, families.cdp_connection_predicate):
+ return _wrap(BrowserCdpConnectionError, exc)
+ if _matches(exc, families.base_error_types, None):
+ return _wrap(BrowserAutomationError, exc)
+ return None
diff --git a/tests/unit/test_exception_messages.py b/tests/unit/test_exception_messages.py
index 8986bb847..d7fd5cfca 100644
--- a/tests/unit/test_exception_messages.py
+++ b/tests/unit/test_exception_messages.py
@@ -91,6 +91,20 @@ def test_unknown_error_redacts_raw_cdp_endpoint_url() -> None:
assert "[remote browser endpoint]" in message
+def test_unknown_error_redacts_http_cdp_discovery_url() -> None:
+ # The /json/version discovery endpoint is reached over http(s) and can carry the vendor host
+ # and a session-bearing token just like the ws socket, so it must be redacted too.
+ inner_exception = Exception(
+ "connect_over_cdp: fetching https://remote.example.internal/json/version?token=SEKRET failed"
+ )
+ message = str(UnknownErrorWhileCreatingBrowserContext("dynamic-browser", inner_exception))
+
+ assert "https://" not in message
+ assert "SEKRET" not in message
+ assert "remote.example.internal" not in message
+ assert "[remote browser endpoint]" in message
+
+
def test_captcha_not_solved_in_time_is_captcha_solve_error() -> None:
# The action handler catches CaptchaSolveError before its generic arm; this
# subclass relationship is what routes captcha-solve failures to the handled
diff --git a/tests/unit/webeye/test_browser_errors.py b/tests/unit/webeye/test_browser_errors.py
new file mode 100644
index 000000000..6cf7d9eb2
--- /dev/null
+++ b/tests/unit/webeye/test_browser_errors.py
@@ -0,0 +1,279 @@
+from __future__ import annotations
+
+import pytest
+from playwright._impl._errors import Error as PWError
+from playwright._impl._errors import TargetClosedError as PWTargetClosedError
+from playwright._impl._errors import TimeoutError as PWTimeoutError
+
+from skyvern.exceptions import SkyvernException
+from skyvern.webeye.browser_errors import (
+ BrowserAutomationError,
+ BrowserCdpConnectionError,
+ BrowserEngineErrorFamilies,
+ BrowserErrorFamiliesConfigError,
+ BrowserRetryableCdpError,
+ BrowserTargetClosedError,
+ BrowserTimeoutError,
+ classify_browser_error,
+)
+
+
+def _stock_families(**overrides: object) -> BrowserEngineErrorFamilies:
+ base: dict[str, object] = {
+ "timeout_types": (PWTimeoutError,),
+ "target_closed_types": (PWTargetClosedError,),
+ "base_error_types": (PWError,),
+ }
+ base.update(overrides)
+ return BrowserEngineErrorFamilies(**base) # type: ignore[arg-type]
+
+
+def test_taxonomy_subclasses_skyvern_exception() -> None:
+ assert issubclass(BrowserAutomationError, SkyvernException)
+ for cls in (
+ BrowserTimeoutError,
+ BrowserTargetClosedError,
+ BrowserCdpConnectionError,
+ BrowserRetryableCdpError,
+ ):
+ assert issubclass(cls, BrowserAutomationError)
+ # Retryable CDP error is a specialization of the transport error.
+ assert issubclass(BrowserRetryableCdpError, BrowserCdpConnectionError)
+
+
+def test_timeout_type_classifies_to_timeout_with_cause() -> None:
+ original = PWTimeoutError("navigation timed out")
+ result = classify_browser_error(original, _stock_families())
+ assert type(result) is BrowserTimeoutError
+ assert result.__cause__ is original
+
+
+def test_target_closed_type_classifies_to_target_closed_with_cause() -> None:
+ original = PWTargetClosedError("Target page, context or browser has been closed")
+ result = classify_browser_error(original, _stock_families())
+ assert type(result) is BrowserTargetClosedError
+ assert result.__cause__ is original
+
+
+def test_transport_classification_via_type_and_predicate() -> None:
+ # Transport/CDP classification is driven entirely by caller-supplied types and
+ # predicates. The prose (message substrings) lives in the test, never in the
+ # production module, proving the classifier is not coupled to a package's wording.
+ class EngineTransportError(PWError):
+ pass
+
+ by_type = classify_browser_error(
+ EngineTransportError("websocket closed"),
+ _stock_families(cdp_connection_types=(EngineTransportError,)),
+ )
+ assert type(by_type) is BrowserCdpConnectionError
+
+ def transport_predicate(exc: BaseException) -> bool:
+ return "econnrefused" in str(exc).lower()
+
+ original = PWError("connect ECONNREFUSED 127.0.0.1:9222")
+ by_predicate = classify_browser_error(
+ original,
+ _stock_families(cdp_connection_predicate=transport_predicate),
+ )
+ assert type(by_predicate) is BrowserCdpConnectionError
+ assert by_predicate.__cause__ is original
+
+ # A generic engine Error with no transport signal falls back to the base taxonomy.
+ generic = classify_browser_error(
+ PWError("some other engine failure"),
+ _stock_families(cdp_connection_predicate=transport_predicate),
+ )
+ assert type(generic) is BrowserAutomationError
+
+
+def test_retryable_classification_and_precedence() -> None:
+ class RetryableTransportError(PWError):
+ pass
+
+ # A retryable type also matches base_error_types (via PWError inheritance);
+ # retryable must win because it is the most specific family.
+ retryable = classify_browser_error(
+ RetryableTransportError("temporary CDP hiccup"),
+ _stock_families(retryable_types=(RetryableTransportError,)),
+ )
+ assert type(retryable) is BrowserRetryableCdpError
+ assert isinstance(retryable, BrowserCdpConnectionError)
+
+ # A retryable predicate takes precedence over a plain cdp-connection type match.
+ original = PWError("connection reset by peer")
+ by_predicate = classify_browser_error(
+ original,
+ BrowserEngineErrorFamilies(
+ cdp_connection_types=(PWError,),
+ retryable_predicate=lambda exc: "reset" in str(exc).lower(),
+ ),
+ )
+ assert type(by_predicate) is BrowserRetryableCdpError
+ assert by_predicate.__cause__ is original
+
+ # Timeout wins over the generic base family even though PWTimeoutError is a PWError.
+ timeout = classify_browser_error(PWTimeoutError("slow"), _stock_families())
+ assert type(timeout) is BrowserTimeoutError
+
+
+def test_predicate_receives_every_otherwise_unmatched_exception() -> None:
+ # Contract pin: predicates are NOT implicitly gated by engine or exception type.
+ # A predicate that keys only on message text will claim an unrelated exception
+ # carrying that text — this is by design, so predicates must scope themselves by
+ # engine identity. If this ever changes to auto-type-gate predicates, this test
+ # must fail loudly rather than the behavior shifting silently.
+ naive_predicate = classify_browser_error(
+ ValueError("connection reset"),
+ BrowserEngineErrorFamilies(retryable_predicate=lambda exc: "reset" in str(exc).lower()),
+ )
+ assert type(naive_predicate) is BrowserRetryableCdpError
+
+ # A predicate that scopes by engine identity leaves foreign exceptions unmatched.
+ scoped_predicate = classify_browser_error(
+ ValueError("connection reset"),
+ BrowserEngineErrorFamilies(
+ retryable_predicate=lambda exc: isinstance(exc, PWError) and "reset" in str(exc).lower()
+ ),
+ )
+ assert scoped_predicate is None
+
+
+class _EngineCdpError(PWError):
+ """Stand-in for a selected engine's native CDP/transport error class."""
+
+
+ "endpoint, secret_fragments",
+ [
+ ("ws://[email protected]:9222/devtools/browser", ("tok-9f3a", "127.0.0.1")),
+ (
+ "wss://user:[email protected]/session?apiKey=SEKRET",
+ ("secret", "SEKRET", "remote.example.internal"),
+ ),
+ ],
+)
+def test_ws_endpoint_urls_redacted_for_every_family(endpoint: str, secret_fragments: tuple[str, ...]) -> None:
+ # A ws/wss devtools socket URL is unambiguously a CDP endpoint, so it is redacted even on a
+ # non-CDP family (here a timeout) — this credentialed socket must never reach the user message.
+ original = PWTimeoutError(f"Timeout 30000ms exceeded connecting to {endpoint}")
+ result = classify_browser_error(original, _stock_families())
+ assert type(result) is BrowserTimeoutError
+ for fragment in secret_fragments:
+ assert fragment not in result.message
+ assert "[remote browser endpoint]" in result.message
+ # Non-sensitive prose stays useful for the user.
+ assert "Timeout 30000ms exceeded" in result.message
+ # The raw, unredacted native error is preserved for internal diagnostics.
+ assert result.__cause__ is original
+ assert endpoint in str(result.__cause__)
+
+
+ "cdp_family, expected_cls",
+ [
+ ({"cdp_connection_types": (_EngineCdpError,)}, BrowserCdpConnectionError),
+ ({"retryable_types": (_EngineCdpError,)}, BrowserRetryableCdpError),
+ ],
+)
+ "endpoint, secret_fragments",
+ [
+ ("http://[email protected]:9222/json/version", ("token", "10.0.0.4")),
+ ("https://remote.example.internal/json/version?token=SEKRET", ("SEKRET", "remote.example.internal")),
+ ],
+)
+def test_http_discovery_url_redacted_only_for_cdp_family(
+ cdp_family: dict[str, object],
+ expected_cls: type[BrowserCdpConnectionError],
+ endpoint: str,
+ secret_fragments: tuple[str, ...],
+) -> None:
+ # In a CDP-connection context an http(s) /json/version discovery URL carries the same vendor
+ # host and session token as the ws socket, so it is redacted for the CDP family (retryable too).
+ original = _EngineCdpError(f"connect_over_cdp: fetching {endpoint} failed")
+ result = classify_browser_error(original, _stock_families(**cdp_family))
+ assert type(result) is expected_cls
+ for fragment in secret_fragments:
+ assert fragment not in result.message
+ assert "[remote browser endpoint]" in result.message
+ assert result.__cause__ is original
+ assert endpoint in str(result.__cause__)
+
+
+def test_navigation_http_url_survives_on_non_cdp_family() -> None:
+ # An ordinary navigation/target http(s) URL is the site the user is automating, not a browser
+ # endpoint — on a non-CDP family it stays intact so it remains useful for debugging.
+ url = "https://customer-portal.example.com/login"
+ original = PWTimeoutError(f"Timeout 30000ms exceeded navigating to {url}")
+ result = classify_browser_error(original, _stock_families())
+ assert type(result) is BrowserTimeoutError
+ assert url in result.message # nosemgrep: incomplete-url-substring-sanitization
+ assert "[remote browser endpoint]" not in result.message
+ assert result.message == str(original)
+
+
+def test_wrapped_message_without_endpoint_url_is_unchanged() -> None:
+ original = PWTimeoutError("navigation timed out after 30000ms")
+ result = classify_browser_error(original, _stock_families())
+ assert result is not None
+ assert result.message == str(original)
+
+
+def test_plain_value_error_returns_none() -> None:
+ assert classify_browser_error(ValueError("not an engine error"), _stock_families()) is None
+ # An unknown error must never be swallowed or flattened into the base taxonomy.
+ assert classify_browser_error(RuntimeError("boom"), _stock_families()) is None
+
+
+def test_cross_engine_distinct_class_identities_are_adapter_bound() -> None:
+ class OtherEngineError(Exception):
+ pass
+
+ class OtherEngineTimeoutError(OtherEngineError):
+ pass
+
+ other_families = BrowserEngineErrorFamilies(
+ timeout_types=(OtherEngineTimeoutError,),
+ base_error_types=(OtherEngineError,),
+ )
+
+ other_timeout = OtherEngineTimeoutError("other engine timed out")
+ # Classified by its own engine's families...
+ assert type(classify_browser_error(other_timeout, other_families)) is BrowserTimeoutError
+ # ...but invisible to the stock engine's families (distinct class identities).
+ assert classify_browser_error(other_timeout, _stock_families()) is None
+ # And the stock engine's timeout is invisible to the other engine's families.
+ assert classify_browser_error(PWTimeoutError("stock timed out"), other_families) is None
+
+
+def test_positive_cases_preserve_original_as_cause() -> None:
+ cases = [
+ (PWTimeoutError("t"), _stock_families(), BrowserTimeoutError),
+ (PWTargetClosedError("c"), _stock_families(), BrowserTargetClosedError),
+ (PWError("generic"), _stock_families(), BrowserAutomationError),
+ ]
+ for original, families, expected in cases:
+ result = classify_browser_error(original, families)
+ assert type(result) is expected
+ assert result.__cause__ is original
+
+
+def test_overlapping_family_configuration_fails_clearly() -> None:
+ with pytest.raises(BrowserErrorFamiliesConfigError) as excinfo:
+ BrowserEngineErrorFamilies(
+ timeout_types=(PWTimeoutError,),
+ target_closed_types=(PWTimeoutError,),
+ )
+ assert "TimeoutError" in str(excinfo.value)
+
+
+def test_duplicate_within_single_family_fails_clearly() -> None:
+ with pytest.raises(BrowserErrorFamiliesConfigError) as excinfo:
+ BrowserEngineErrorFamilies(timeout_types=(PWTimeoutError, PWTimeoutError))
+ assert "twice in timeout_types" in str(excinfo.value)
+
+
+def test_non_exception_family_entry_fails_clearly() -> None:
+ with pytest.raises(BrowserErrorFamiliesConfigError):
+ BrowserEngineErrorFamilies(timeout_types=(str,)) # type: ignore[arg-type]