Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
[SKY-12007] Add engine-neutral browser error taxonomy (#7468)
LawyZheng6b0b4fdc77d2
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.pyindex 2031e1ff6..6d6999917 100644--- a/skyvern/exceptions.py+++ b/skyvern/exceptions.py@@ -136,13 +136,22 @@ def _is_browser_connection_error(message: str) -> bool:# A raw CDP connect failure (e.g. from playwright.chromium.connect_over_cdp) echoes the-# ws/wss endpoint URL, which can carry the remote-browser vendor host, a session-bearing-# path/query, or credentials embedded as user:pass@host. Redact it before it reaches a user.-_CDP_WS_URL_RE = re.compile(r"wss?://\S+", re.IGNORECASE)+# endpoint URL, which can carry the remote-browser vendor host, a session-bearing path/query,+# or credentials embedded as user:pass@host. The devtools socket is always ws/wss, so a ws/wss+# URL in a browser error is unambiguously a CDP endpoint and safe to redact anywhere. The+# /json/version discovery endpoint is http/https and carries the same host/token, but so does an+# ordinary navigation/target/proxy URL — an http(s) URL is only known to be a CDP endpoint in a+# CDP-connection context, so http(s) redaction is scoped to that context (see redact_cdp_endpoint_urls).+_WS_ENDPOINT_URL_RE = re.compile(r"wss?://\S+", re.IGNORECASE)+_CDP_ENDPOINT_URL_RE = re.compile(r"(?:wss?|https?)://\S+", re.IGNORECASE)-def _redact_cdp_endpoint_urls(message: str) -> str:- return _CDP_WS_URL_RE.sub("[remote browser endpoint]", message)+def redact_ws_endpoint_urls(message: str) -> str:+ return _WS_ENDPOINT_URL_RE.sub("[remote browser endpoint]", message)+++def redact_cdp_endpoint_urls(message: str) -> str:+ return _CDP_ENDPOINT_URL_RE.sub("[remote browser endpoint]", message)def get_user_facing_exception_message(exception: Exception) -> str:@@ -506,7 +515,7 @@ class UnknownErrorWhileCreatingBrowserContext(SkyvernException):if isinstance(exception, CdpConnectionConfigurationError):return exception.message or str(exception)- raw_message = _redact_cdp_endpoint_urls(str(exception).strip())+ raw_message = redact_cdp_endpoint_urls(str(exception).strip())raw_lower = raw_message.lower()# Browser launch environment errors: worker cannot initialize thediff --git a/skyvern/webeye/browser_errors.py b/skyvern/webeye/browser_errors.pynew file mode 100644index 000000000..e77295a16--- /dev/null+++ b/skyvern/webeye/browser_errors.py@@ -0,0 +1,134 @@+"""Engine-neutral browser-error taxonomy and classifier.++Skyvern can select among several browser engines (for example stock Playwright or a+private engine), each of which raises its own native error classes with distinct+identities. This module owns a taxonomy that recovery code can branch on without+importing any engine, plus a classifier parameterized by the *selected* engine's+native error families. It deliberately imports no browser driver at module scope so+it stays driver-agnostic and reusable by every adapter.+"""++from __future__ import annotations++from collections.abc import Callable+from dataclasses import dataclass++from skyvern.exceptions import SkyvernException, redact_cdp_endpoint_urls, redact_ws_endpoint_urls++ExceptionType = type[BaseException]+ErrorPredicate = Callable[[BaseException], bool]+++class BrowserAutomationError(SkyvernException):+ """Engine-neutral base for a browser failure classified from a driver-native error."""+++class BrowserTimeoutError(BrowserAutomationError):+ """A driver operation exceeded its deadline."""+++class BrowserTargetClosedError(BrowserAutomationError):+ """The target page, frame, context, or browser was closed underneath an operation."""+++class BrowserCdpConnectionError(BrowserAutomationError):+ """A CDP/transport-level failure talking to the browser."""+++class BrowserRetryableCdpError(BrowserCdpConnectionError):+ """A CDP/transport failure the caller may retry (e.g. a transient disconnect)."""+++class BrowserErrorFamiliesConfigError(ValueError):+ """Raised when an engine's error-family configuration is invalid or ambiguous."""+++_FAMILY_TYPE_FIELDS = (+ "timeout_types",+ "target_closed_types",+ "cdp_connection_types",+ "retryable_types",+ "base_error_types",+)+++@dataclass(frozen=True)+class BrowserEngineErrorFamilies:+ """Native error classes/predicates for one selected engine, grouped by taxonomy; a native type+ appearing in more than one family is rejected at construction time (rather than resolved by+ isinstance ordering) to keep classification unambiguous, and the ``cdp_connection_predicate`` /+ ``retryable_predicate`` run on *every* exception that reaches their step unmatched by an earlier+ family's types, so a predicate must scope itself by the engine's identity (type-check, or match+ transport signals only that engine emits) — one keyed solely on message substrings will also+ claim unrelated exceptions carrying the same text.+ """++ timeout_types: tuple[ExceptionType, ...] = ()+ target_closed_types: tuple[ExceptionType, ...] = ()+ cdp_connection_types: tuple[ExceptionType, ...] = ()+ retryable_types: tuple[ExceptionType, ...] = ()+ base_error_types: tuple[ExceptionType, ...] = ()+ cdp_connection_predicate: ErrorPredicate | None = None+ retryable_predicate: ErrorPredicate | None = None++ def __post_init__(self) -> None:+ seen: dict[ExceptionType, str] = {}+ for field_name in _FAMILY_TYPE_FIELDS:+ for entry in getattr(self, field_name):+ if not (isinstance(entry, type) and issubclass(entry, BaseException)):+ raise BrowserErrorFamiliesConfigError(+ f"{field_name} entry {entry!r} is not a BaseException subclass"+ )+ prior = seen.get(entry)+ if prior is not None:+ location = f"twice in {field_name}" if prior == field_name else f"in both {prior} and {field_name}"+ raise BrowserErrorFamiliesConfigError(+ f"{entry.__name__} appears {location}; a native error type may only belong to one family"+ )+ seen[entry] = field_name+++def _matches(exc: BaseException, types: tuple[ExceptionType, ...], predicate: ErrorPredicate | None) -> bool:+ if types and isinstance(exc, types):+ return True+ return predicate is not None and predicate(exc)+++def _wrap(cls: type[BrowserAutomationError], exc: BaseException) -> BrowserAutomationError:+ """Wrap a native error as ``cls``, redacting browser-endpoint URLs from ``.message`` while keeping+ the raw native error on ``__cause__`` for logs and diagnostics. A ws/wss devtools socket URL is+ unambiguously a CDP endpoint, so it is redacted for every family; an http/https URL is only known+ to be a CDP discovery endpoint (rather than an ordinary navigation/target/proxy URL the user wants+ to see) when ``cls`` is a CDP-connection family, so http(s) redaction is scoped there.+ """+ raw = str(exc)+ redacted = (+ redact_cdp_endpoint_urls(raw) if issubclass(cls, BrowserCdpConnectionError) else redact_ws_endpoint_urls(raw)+ )+ error = cls(redacted)+ error.__cause__ = exc+ return error+++def classify_browser_error(+ exc: BaseException,+ families: BrowserEngineErrorFamilies,+) -> BrowserAutomationError | None:+ """Map a native engine error to the taxonomy, or return ``None`` if unrecognized. Precedence is+ most-specific first — retryable CDP, target closed, timeout, CDP transport, then the engine's+ generic base error — so a type matching several families via inheritance (e.g. a timeout that+ subclasses the base error) resolves to the earliest match; ``None`` means the error is not in+ this engine's taxonomy and the caller must re-raise it (never swallowed or flattened into the+ base type), and positive classifications preserve the original native exception as ``__cause__``.+ """+ if _matches(exc, families.retryable_types, families.retryable_predicate):+ return _wrap(BrowserRetryableCdpError, exc)+ if _matches(exc, families.target_closed_types, None):+ return _wrap(BrowserTargetClosedError, exc)+ if _matches(exc, families.timeout_types, None):+ return _wrap(BrowserTimeoutError, exc)+ if _matches(exc, families.cdp_connection_types, families.cdp_connection_predicate):+ return _wrap(BrowserCdpConnectionError, exc)+ if _matches(exc, families.base_error_types, None):+ return _wrap(BrowserAutomationError, exc)+ return Nonediff --git a/tests/unit/test_exception_messages.py b/tests/unit/test_exception_messages.pyindex 8986bb847..d7fd5cfca 100644--- a/tests/unit/test_exception_messages.py+++ b/tests/unit/test_exception_messages.py@@ -91,6 +91,20 @@ def test_unknown_error_redacts_raw_cdp_endpoint_url() -> None:assert "[remote browser endpoint]" in message+def test_unknown_error_redacts_http_cdp_discovery_url() -> None:+ # The /json/version discovery endpoint is reached over http(s) and can carry the vendor host+ # and a session-bearing token just like the ws socket, so it must be redacted too.+ inner_exception = Exception(+ "connect_over_cdp: fetching https://remote.example.internal/json/version?token=SEKRET failed"+ )+ message = str(UnknownErrorWhileCreatingBrowserContext("dynamic-browser", inner_exception))++ assert "https://" not in message+ assert "SEKRET" not in message+ assert "remote.example.internal" not in message+ assert "[remote browser endpoint]" in message++def test_captcha_not_solved_in_time_is_captcha_solve_error() -> None:# The action handler catches CaptchaSolveError before its generic arm; this# subclass relationship is what routes captcha-solve failures to the handleddiff --git a/tests/unit/webeye/test_browser_errors.py b/tests/unit/webeye/test_browser_errors.pynew file mode 100644index 000000000..6cf7d9eb2--- /dev/null+++ b/tests/unit/webeye/test_browser_errors.py@@ -0,0 +1,279 @@+from __future__ import annotations++import pytest+from playwright._impl._errors import Error as PWError+from playwright._impl._errors import TargetClosedError as PWTargetClosedError+from playwright._impl._errors import TimeoutError as PWTimeoutError++from skyvern.exceptions import SkyvernException+from skyvern.webeye.browser_errors import (+ BrowserAutomationError,+ BrowserCdpConnectionError,+ BrowserEngineErrorFamilies,+ BrowserErrorFamiliesConfigError,+ BrowserRetryableCdpError,+ BrowserTargetClosedError,+ BrowserTimeoutError,+ classify_browser_error,+)+++def _stock_families(**overrides: object) -> BrowserEngineErrorFamilies:+ base: dict[str, object] = {+ "timeout_types": (PWTimeoutError,),+ "target_closed_types": (PWTargetClosedError,),+ "base_error_types": (PWError,),+ }+ base.update(overrides)+ return BrowserEngineErrorFamilies(**base) # type: ignore[arg-type]+++def test_taxonomy_subclasses_skyvern_exception() -> None:+ assert issubclass(BrowserAutomationError, SkyvernException)+ for cls in (+ BrowserTimeoutError,+ BrowserTargetClosedError,+ BrowserCdpConnectionError,+ BrowserRetryableCdpError,+ ):+ assert issubclass(cls, BrowserAutomationError)+ # Retryable CDP error is a specialization of the transport error.+ assert issubclass(BrowserRetryableCdpError, BrowserCdpConnectionError)+++def test_timeout_type_classifies_to_timeout_with_cause() -> None:+ original = PWTimeoutError("navigation timed out")+ result = classify_browser_error(original, _stock_families())+ assert type(result) is BrowserTimeoutError+ assert result.__cause__ is original+++def test_target_closed_type_classifies_to_target_closed_with_cause() -> None:+ original = PWTargetClosedError("Target page, context or browser has been closed")+ result = classify_browser_error(original, _stock_families())+ assert type(result) is BrowserTargetClosedError+ assert result.__cause__ is original+++def test_transport_classification_via_type_and_predicate() -> None:+ # Transport/CDP classification is driven entirely by caller-supplied types and+ # predicates. The prose (message substrings) lives in the test, never in the+ # production module, proving the classifier is not coupled to a package's wording.+ class EngineTransportError(PWError):+ pass++ by_type = classify_browser_error(+ EngineTransportError("websocket closed"),+ _stock_families(cdp_connection_types=(EngineTransportError,)),+ )+ assert type(by_type) is BrowserCdpConnectionError++ def transport_predicate(exc: BaseException) -> bool:+ return "econnrefused" in str(exc).lower()++ original = PWError("connect ECONNREFUSED 127.0.0.1:9222")+ by_predicate = classify_browser_error(+ original,+ _stock_families(cdp_connection_predicate=transport_predicate),+ )+ assert type(by_predicate) is BrowserCdpConnectionError+ assert by_predicate.__cause__ is original++ # A generic engine Error with no transport signal falls back to the base taxonomy.+ generic = classify_browser_error(+ PWError("some other engine failure"),+ _stock_families(cdp_connection_predicate=transport_predicate),+ )+ assert type(generic) is BrowserAutomationError+++def test_retryable_classification_and_precedence() -> None:+ class RetryableTransportError(PWError):+ pass++ # A retryable type also matches base_error_types (via PWError inheritance);+ # retryable must win because it is the most specific family.+ retryable = classify_browser_error(+ RetryableTransportError("temporary CDP hiccup"),+ _stock_families(retryable_types=(RetryableTransportError,)),+ )+ assert type(retryable) is BrowserRetryableCdpError+ assert isinstance(retryable, BrowserCdpConnectionError)++ # A retryable predicate takes precedence over a plain cdp-connection type match.+ original = PWError("connection reset by peer")+ by_predicate = classify_browser_error(+ original,+ BrowserEngineErrorFamilies(+ cdp_connection_types=(PWError,),+ retryable_predicate=lambda exc: "reset" in str(exc).lower(),+ ),+ )+ assert type(by_predicate) is BrowserRetryableCdpError+ assert by_predicate.__cause__ is original++ # Timeout wins over the generic base family even though PWTimeoutError is a PWError.+ timeout = classify_browser_error(PWTimeoutError("slow"), _stock_families())+ assert type(timeout) is BrowserTimeoutError+++def test_predicate_receives_every_otherwise_unmatched_exception() -> None:+ # Contract pin: predicates are NOT implicitly gated by engine or exception type.+ # A predicate that keys only on message text will claim an unrelated exception+ # carrying that text — this is by design, so predicates must scope themselves by+ # engine identity. If this ever changes to auto-type-gate predicates, this test+ # must fail loudly rather than the behavior shifting silently.+ naive_predicate = classify_browser_error(+ ValueError("connection reset"),+ BrowserEngineErrorFamilies(retryable_predicate=lambda exc: "reset" in str(exc).lower()),+ )+ assert type(naive_predicate) is BrowserRetryableCdpError++ # A predicate that scopes by engine identity leaves foreign exceptions unmatched.+ scoped_predicate = classify_browser_error(+ ValueError("connection reset"),+ BrowserEngineErrorFamilies(+ retryable_predicate=lambda exc: isinstance(exc, PWError) and "reset" in str(exc).lower()+ ),+ )+ assert scoped_predicate is None+++class _EngineCdpError(PWError):+ """Stand-in for a selected engine's native CDP/transport error class."""+++ "endpoint, secret_fragments",+ [+ ("ws://[email protected]:9222/devtools/browser", ("tok-9f3a", "127.0.0.1")),+ (+ "wss://user:[email protected]/session?apiKey=SEKRET",+ ("secret", "SEKRET", "remote.example.internal"),+ ),+ ],+)+def test_ws_endpoint_urls_redacted_for_every_family(endpoint: str, secret_fragments: tuple[str, ...]) -> None:+ # A ws/wss devtools socket URL is unambiguously a CDP endpoint, so it is redacted even on a+ # non-CDP family (here a timeout) — this credentialed socket must never reach the user message.+ original = PWTimeoutError(f"Timeout 30000ms exceeded connecting to {endpoint}")+ result = classify_browser_error(original, _stock_families())+ assert type(result) is BrowserTimeoutError+ for fragment in secret_fragments:+ assert fragment not in result.message+ assert "[remote browser endpoint]" in result.message+ # Non-sensitive prose stays useful for the user.+ assert "Timeout 30000ms exceeded" in result.message+ # The raw, unredacted native error is preserved for internal diagnostics.+ assert result.__cause__ is original+ assert endpoint in str(result.__cause__)+++ "cdp_family, expected_cls",+ [+ ({"cdp_connection_types": (_EngineCdpError,)}, BrowserCdpConnectionError),+ ({"retryable_types": (_EngineCdpError,)}, BrowserRetryableCdpError),+ ],+)+ "endpoint, secret_fragments",+ [+ ("http://[email protected]:9222/json/version", ("token", "10.0.0.4")),+ ("https://remote.example.internal/json/version?token=SEKRET", ("SEKRET", "remote.example.internal")),+ ],+)+def test_http_discovery_url_redacted_only_for_cdp_family(+ cdp_family: dict[str, object],+ expected_cls: type[BrowserCdpConnectionError],+ endpoint: str,+ secret_fragments: tuple[str, ...],+) -> None:+ # In a CDP-connection context an http(s) /json/version discovery URL carries the same vendor+ # host and session token as the ws socket, so it is redacted for the CDP family (retryable too).+ original = _EngineCdpError(f"connect_over_cdp: fetching {endpoint} failed")+ result = classify_browser_error(original, _stock_families(**cdp_family))+ assert type(result) is expected_cls+ for fragment in secret_fragments:+ assert fragment not in result.message+ assert "[remote browser endpoint]" in result.message+ assert result.__cause__ is original+ assert endpoint in str(result.__cause__)+++def test_navigation_http_url_survives_on_non_cdp_family() -> None:+ # An ordinary navigation/target http(s) URL is the site the user is automating, not a browser+ # endpoint — on a non-CDP family it stays intact so it remains useful for debugging.+ url = "https://customer-portal.example.com/login"+ original = PWTimeoutError(f"Timeout 30000ms exceeded navigating to {url}")+ result = classify_browser_error(original, _stock_families())+ assert type(result) is BrowserTimeoutError+ assert url in result.message # nosemgrep: incomplete-url-substring-sanitization+ assert "[remote browser endpoint]" not in result.message+ assert result.message == str(original)+++def test_wrapped_message_without_endpoint_url_is_unchanged() -> None:+ original = PWTimeoutError("navigation timed out after 30000ms")+ result = classify_browser_error(original, _stock_families())+ assert result is not None+ assert result.message == str(original)+++def test_plain_value_error_returns_none() -> None:+ assert classify_browser_error(ValueError("not an engine error"), _stock_families()) is None+ # An unknown error must never be swallowed or flattened into the base taxonomy.+ assert classify_browser_error(RuntimeError("boom"), _stock_families()) is None+++def test_cross_engine_distinct_class_identities_are_adapter_bound() -> None:+ class OtherEngineError(Exception):+ pass++ class OtherEngineTimeoutError(OtherEngineError):+ pass++ other_families = BrowserEngineErrorFamilies(+ timeout_types=(OtherEngineTimeoutError,),+ base_error_types=(OtherEngineError,),+ )++ other_timeout = OtherEngineTimeoutError("other engine timed out")+ # Classified by its own engine's families...+ assert type(classify_browser_error(other_timeout, other_families)) is BrowserTimeoutError+ # ...but invisible to the stock engine's families (distinct class identities).+ assert classify_browser_error(other_timeout, _stock_families()) is None+ # And the stock engine's timeout is invisible to the other engine's families.+ assert classify_browser_error(PWTimeoutError("stock timed out"), other_families) is None+++def test_positive_cases_preserve_original_as_cause() -> None:+ cases = [+ (PWTimeoutError("t"), _stock_families(), BrowserTimeoutError),+ (PWTargetClosedError("c"), _stock_families(), BrowserTargetClosedError),+ (PWError("generic"), _stock_families(), BrowserAutomationError),+ ]+ for original, families, expected in cases:+ result = classify_browser_error(original, families)+ assert type(result) is expected+ assert result.__cause__ is original+++def test_overlapping_family_configuration_fails_clearly() -> None:+ with pytest.raises(BrowserErrorFamiliesConfigError) as excinfo:+ BrowserEngineErrorFamilies(+ timeout_types=(PWTimeoutError,),+ target_closed_types=(PWTimeoutError,),+ )+ assert "TimeoutError" in str(excinfo.value)+++def test_duplicate_within_single_family_fails_clearly() -> None:+ with pytest.raises(BrowserErrorFamiliesConfigError) as excinfo:+ BrowserEngineErrorFamilies(timeout_types=(PWTimeoutError, PWTimeoutError))+ assert "twice in timeout_types" in str(excinfo.value)+++def test_non_exception_family_entry_fails_clearly() -> None:+ with pytest.raises(BrowserErrorFamiliesConfigError):+ BrowserEngineErrorFamilies(timeout_types=(str,)) # type: ignore[arg-type]