Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-11934): sanitize remaining MCP ApiError leak paths (#7384)
Marc Kelechava187f9be98f28
diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.pyindex 718eef6c3..eb1aba17b 100644--- a/skyvern/cli/mcp_tools/browser.py+++ b/skyvern/cli/mcp_tools/browser.py@@ -129,7 +129,7 @@ def _exception_message(exc: Exception) -> str:# get_user_facing_exception_message's "Unexpected error: {exception}" fallback in# skyvern/exceptions.py) — never surface them. 4xx bodies are the API's intended# client-facing feedback (typed BadRequest/NotFound/UnprocessableEntity errors).- surface_body = not (isinstance(status_code, int) and status_code >= 500)+ surface_body = status_code is None or (isinstance(status_code, int) and 400 <= status_code < 500)body_message = _message_from_error_body(getattr(exc, "body", None)) if surface_body else Noneif body_message:return f"HTTP {status_code}: {body_message}" if status_code is not None else body_message@@ -526,8 +526,9 @@ async def skyvern_drag(timing_ms=timer.timing_ms,error=make_error(ErrorCode.SELECTOR_NOT_FOUND,- str(e),+ _exception_message(e),"Verify source and target selectors match elements on the page",+ details=_exception_details(e),),)except Exception as e:@@ -545,7 +546,12 @@ async def skyvern_drag(ok=False,browser_context=ctx,timing_ms=timer.timing_ms,- error=make_error(ErrorCode.ACTION_FAILED, str(e), "The drag operation failed"),+ error=make_error(+ ErrorCode.ACTION_FAILED,+ _exception_message(e),+ "The drag operation failed",+ details=_exception_details(e),+ ),)return make_result(@@ -1301,7 +1307,12 @@ async def skyvern_press_key(ok=False,browser_context=ctx,timing_ms=timer.timing_ms,- error=make_error(ErrorCode.ACTION_FAILED, str(e), "Check key name is valid"),+ error=make_error(+ ErrorCode.ACTION_FAILED,+ _exception_message(e),+ "Check key name is valid",+ details=_exception_details(e),+ ),)if selector and intent:@@ -1391,7 +1402,11 @@ async def skyvern_wait(breakif loop.time() >= deadline:code = ErrorCode.SDK_ERROR if last_error else ErrorCode.TIMEOUT- msg = str(last_error) if last_error else f"Condition not met within {timeout}ms: {intent}"+ msg = (+ _exception_message(last_error)+ if last_error+ else f"Condition not met within {timeout}ms: {intent}"+ )return make_result("skyvern_wait",ok=False,@@ -1401,6 +1416,7 @@ async def skyvern_wait(code,msg,"Increase timeout or check that the condition can be satisfied",+ details=_exception_details(last_error) if last_error else None,),)await page.wait_for_timeout(poll_interval_ms)@@ -1415,7 +1431,12 @@ async def skyvern_wait(ok=False,browser_context=ctx,timing_ms=timer.timing_ms,- error=make_error(ErrorCode.TIMEOUT, str(e), "Condition was not met within timeout"),+ error=make_error(+ ErrorCode.TIMEOUT,+ _exception_message(e),+ "Condition was not met within timeout",+ details=_exception_details(e),+ ),)sdk_eq = ""diff --git a/tests/unit/_mcp_browser_fakes.py b/tests/unit/_mcp_browser_fakes.pyindex 1a5181972..a649b1ae3 100644--- a/tests/unit/_mcp_browser_fakes.py+++ b/tests/unit/_mcp_browser_fakes.py@@ -1,5 +1,6 @@from __future__ import annotations+import asynciofrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import AsyncMock, MagicMock@@ -10,6 +11,15 @@ from skyvern.cli.core.result import BrowserContextfrom skyvern.cli.core.session_manager import SessionState+def make_real_wait_for_timeout() -> AsyncMock:+ """A `page.wait_for_timeout` fake that actually sleeps, for tests driving real deadline loops."""++ async def _wait(delay_ms: int) -> None:+ await asyncio.sleep(delay_ms / 1000)++ return AsyncMock(side_effect=_wait)++def make_session_state(**overrides: Any) -> SessionState:state = SessionState()for key, value in overrides.items():diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.pyindex c84a50545..f91a0359b 100644--- a/tests/unit/test_mcp_browser_tools.py+++ b/tests/unit/test_mcp_browser_tools.py@@ -12,7 +12,7 @@ from skyvern.cli.core.result import BrowserContext, set_concise_responsesfrom skyvern.cli.mcp_tools import browser as mcp_browserfrom skyvern.cli.mcp_tools import mcpfrom skyvern.client.errors import InternalServerError, UnprocessableEntityError-from tests.unit._mcp_browser_fakes import make_probe_locator+from tests.unit._mcp_browser_fakes import make_probe_locator, make_real_wait_for_timeout@pytest.mark.asyncio@@ -537,6 +537,26 @@ async def test_skyvern_press_key_intent_keeps_30s_default_timeout(monkeypatch: ppress.assert_awaited_once_with("Enter", timeout=30000)+async def test_skyvern_press_key_intent_error_does_not_leak_headers(monkeypatch: pytest.MonkeyPatch) -> None:+ press = AsyncMock(+ side_effect=InternalServerError(+ body={"error": "Unexpected error: sk-BODY-SECRET"},+ headers={"authorization": "Bearer sk-SECRET"},+ )+ )+ _action_page(monkeypatch, locator=MagicMock(return_value=SimpleNamespace(press=press)))++ result = await mcp_browser.skyvern_press_key(key="Enter", intent="the search box")++ assert result["ok"] is False+ message = result["error"]["message"]+ assert message == "HTTP 500: InternalServerError"+ assert result["error"]["details"] == {"exception_type": "InternalServerError", "status_code": 500}+ for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):+ assert leaked not in message.lower()++@pytest.mark.asyncioasync def test_skyvern_press_key_direct_failure_reports_element_state(monkeypatch: pytest.MonkeyPatch) -> None:probe_locator = make_probe_locator(count=1, visible=False)@@ -791,6 +811,16 @@ async def test_skyvern_click_native_option_selector_selects_by_index(monkeypatchassert result["data"]["sdk_equivalent"] == 'await page.select_option("#region", index=3)'+def test_exception_message_suppresses_body_outside_4xx() -> None:+ class NonErrorStatusException(Exception):+ status_code = 200+ body = {"detail": "should not be surfaced"}++ message = mcp_browser._exception_message(NonErrorStatusException())++ assert "should not be surfaced" not in message++@pytest.mark.asyncioasync def test_skyvern_type_ai_error_surfaces_http_status_and_body(monkeypatch: pytest.MonkeyPatch) -> None:# 4xx bodies are the API's intended client-facing feedback — surfaced for self-correction.@@ -887,6 +917,58 @@ async def test_skyvern_act_sdk_error_does_not_leak_headers(monkeypatch: pytest.Massert leaked not in message.lower()+async def test_skyvern_drag_ai_error_does_not_leak_headers(monkeypatch: pytest.MonkeyPatch) -> None:+ _action_page(monkeypatch)+ monkeypatch.setattr(+ mcp_browser,+ "do_act",+ AsyncMock(+ side_effect=InternalServerError(+ body={"error": "Unexpected error: sk-BODY-SECRET"},+ headers={"authorization": "Bearer sk-SECRET"},+ )+ ),+ )++ result = await mcp_browser.skyvern_drag(source_intent="the card", target_intent="the trash bin")++ assert result["ok"] is False+ message = result["error"]["message"]+ assert message == "HTTP 500: InternalServerError"+ assert result["error"]["details"] == {"exception_type": "InternalServerError", "status_code": 500}+ for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):+ assert leaked not in message.lower()+++async def test_skyvern_wait_intent_error_does_not_leak_headers(monkeypatch: pytest.MonkeyPatch) -> None:+ _action_page(+ monkeypatch,+ validate=AsyncMock(+ side_effect=InternalServerError(+ body={"error": "Unexpected error: sk-BODY-SECRET"},+ headers={"authorization": "Bearer sk-SECRET"},+ )+ ),+ wait_for_timeout=make_real_wait_for_timeout(),+ )++ result = await mcp_browser.skyvern_wait(+ intent="the spinner disappears",+ timeout=1000,+ poll_interval_ms=500,+ )++ assert result["ok"] is False+ assert result["error"]["code"] == mcp_browser.ErrorCode.SDK_ERROR+ message = result["error"]["message"]+ assert message == "HTTP 500: InternalServerError"+ assert result["error"]["details"] == {"exception_type": "InternalServerError", "status_code": 500}+ for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):+ assert leaked not in message.lower()++@pytest.mark.asyncioasync def test_skyvern_type_ai_error_uses_exception_type_for_empty_message(monkeypatch: pytest.MonkeyPatch,diff --git a/tests/unit/test_mcp_observe_execute.py b/tests/unit/test_mcp_observe_execute.pyindex 10d1c5f0e..eb4b1ef41 100644--- a/tests/unit/test_mcp_observe_execute.py+++ b/tests/unit/test_mcp_observe_execute.py@@ -24,7 +24,8 @@ from skyvern.cli.core.result import BrowserContextfrom skyvern.cli.core.session_manager import scoped_sessionfrom skyvern.cli.mcp_tools import browser as mcp_browserfrom skyvern.cli.mcp_tools import tabs as mcp_tabs-from tests.unit._mcp_browser_fakes import make_session_state+from skyvern.client.errors import InternalServerError+from tests.unit._mcp_browser_fakes import make_real_wait_for_timeout, make_session_state# ---------------------------------------------------------------------------# Helpers@@ -1472,6 +1473,40 @@ class TestSkyvernExecuteMCP:assert step_error["details"]["element_state"] == "hidden"assert step_error["details"]["selector"] == "#field"+ @pytest.mark.asyncio+ async def test_execute_wait_error_does_not_leak_headers(self, monkeypatch: pytest.MonkeyPatch) -> None:+ page = _make_page()+ page.validate = AsyncMock(+ side_effect=InternalServerError(+ body={"error": "Unexpected error: sk-BODY-SECRET"},+ headers={"authorization": "Bearer sk-SECRET"},+ )+ )+ page.wait_for_timeout = make_real_wait_for_timeout()+ ctx = BrowserContext(mode="local")+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))++ result = await mcp_browser.skyvern_execute(+ steps=[+ {+ "tool": "wait",+ "params": {+ "intent": "the spinner disappears",+ "timeout": 1000,+ "poll_interval_ms": 500,+ },+ }+ ]+ )++ assert result["ok"] is False+ step_error = result["data"]["results"][0]["error"]+ assert step_error["message"] == "HTTP 500: InternalServerError"+ assert step_error["details"] == {"exception_type": "InternalServerError", "status_code": 500}+ message = step_error["message"].lower()+ for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):+ assert leaked not in message+@pytest.mark.asyncioasync def test_execute_ref_to_unnamed_duplicate_uses_nth(self, monkeypatch: pytest.MonkeyPatch) -> None:"""SKY-9701: clicking ref=e2 on a page with 4 unnamed comboboxes resolves to the 3rd, not the 1st."""