Skip to content

Automate browser based workflows with AI

fix(SKY-11934): sanitize remaining MCP ApiError leak paths (#7384)

Marc Kelechava187f9be98f28
diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.py
index 718eef6c3..eb1aba17b 100644
--- a/skyvern/cli/mcp_tools/browser.py
+++ b/skyvern/cli/mcp_tools/browser.py
@@ -129,7 +129,7 @@ def _exception_message(exc: Exception) -> str:
# get_user_facing_exception_message's "Unexpected error: {exception}" fallback in
# skyvern/exceptions.py) — never surface them. 4xx bodies are the API's intended
# client-facing feedback (typed BadRequest/NotFound/UnprocessableEntity errors).
- surface_body = not (isinstance(status_code, int) and status_code >= 500)
+ surface_body = status_code is None or (isinstance(status_code, int) and 400 <= status_code < 500)
body_message = _message_from_error_body(getattr(exc, "body", None)) if surface_body else None
if body_message:
return f"HTTP {status_code}: {body_message}" if status_code is not None else body_message
@@ -526,8 +526,9 @@ async def skyvern_drag(
timing_ms=timer.timing_ms,
error=make_error(
ErrorCode.SELECTOR_NOT_FOUND,
- str(e),
+ _exception_message(e),
"Verify source and target selectors match elements on the page",
+ details=_exception_details(e),
),
)
except Exception as e:
@@ -545,7 +546,12 @@ async def skyvern_drag(
ok=False,
browser_context=ctx,
timing_ms=timer.timing_ms,
- error=make_error(ErrorCode.ACTION_FAILED, str(e), "The drag operation failed"),
+ error=make_error(
+ ErrorCode.ACTION_FAILED,
+ _exception_message(e),
+ "The drag operation failed",
+ details=_exception_details(e),
+ ),
)
return make_result(
@@ -1301,7 +1307,12 @@ async def skyvern_press_key(
ok=False,
browser_context=ctx,
timing_ms=timer.timing_ms,
- error=make_error(ErrorCode.ACTION_FAILED, str(e), "Check key name is valid"),
+ error=make_error(
+ ErrorCode.ACTION_FAILED,
+ _exception_message(e),
+ "Check key name is valid",
+ details=_exception_details(e),
+ ),
)
if selector and intent:
@@ -1391,7 +1402,11 @@ async def skyvern_wait(
break
if loop.time() >= deadline:
code = ErrorCode.SDK_ERROR if last_error else ErrorCode.TIMEOUT
- msg = str(last_error) if last_error else f"Condition not met within {timeout}ms: {intent}"
+ msg = (
+ _exception_message(last_error)
+ if last_error
+ else f"Condition not met within {timeout}ms: {intent}"
+ )
return make_result(
"skyvern_wait",
ok=False,
@@ -1401,6 +1416,7 @@ async def skyvern_wait(
code,
msg,
"Increase timeout or check that the condition can be satisfied",
+ details=_exception_details(last_error) if last_error else None,
),
)
await page.wait_for_timeout(poll_interval_ms)
@@ -1415,7 +1431,12 @@ async def skyvern_wait(
ok=False,
browser_context=ctx,
timing_ms=timer.timing_ms,
- error=make_error(ErrorCode.TIMEOUT, str(e), "Condition was not met within timeout"),
+ error=make_error(
+ ErrorCode.TIMEOUT,
+ _exception_message(e),
+ "Condition was not met within timeout",
+ details=_exception_details(e),
+ ),
)
sdk_eq = ""
diff --git a/tests/unit/_mcp_browser_fakes.py b/tests/unit/_mcp_browser_fakes.py
index 1a5181972..a649b1ae3 100644
--- a/tests/unit/_mcp_browser_fakes.py
+++ b/tests/unit/_mcp_browser_fakes.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock
@@ -10,6 +11,15 @@ from skyvern.cli.core.result import BrowserContext
from skyvern.cli.core.session_manager import SessionState
+def make_real_wait_for_timeout() -> AsyncMock:
+ """A `page.wait_for_timeout` fake that actually sleeps, for tests driving real deadline loops."""
+
+ async def _wait(delay_ms: int) -> None:
+ await asyncio.sleep(delay_ms / 1000)
+
+ return AsyncMock(side_effect=_wait)
+
+
def make_session_state(**overrides: Any) -> SessionState:
state = SessionState()
for key, value in overrides.items():
diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.py
index c84a50545..f91a0359b 100644
--- a/tests/unit/test_mcp_browser_tools.py
+++ b/tests/unit/test_mcp_browser_tools.py
@@ -12,7 +12,7 @@ from skyvern.cli.core.result import BrowserContext, set_concise_responses
from skyvern.cli.mcp_tools import browser as mcp_browser
from skyvern.cli.mcp_tools import mcp
from skyvern.client.errors import InternalServerError, UnprocessableEntityError
-from tests.unit._mcp_browser_fakes import make_probe_locator
+from tests.unit._mcp_browser_fakes import make_probe_locator, make_real_wait_for_timeout
@pytest.mark.asyncio
@@ -537,6 +537,26 @@ async def test_skyvern_press_key_intent_keeps_30s_default_timeout(monkeypatch: p
press.assert_awaited_once_with("Enter", timeout=30000)
+async def test_skyvern_press_key_intent_error_does_not_leak_headers(monkeypatch: pytest.MonkeyPatch) -> None:
+ press = AsyncMock(
+ side_effect=InternalServerError(
+ body={"error": "Unexpected error: sk-BODY-SECRET"},
+ headers={"authorization": "Bearer sk-SECRET"},
+ )
+ )
+ _action_page(monkeypatch, locator=MagicMock(return_value=SimpleNamespace(press=press)))
+
+ result = await mcp_browser.skyvern_press_key(key="Enter", intent="the search box")
+
+ assert result["ok"] is False
+ message = result["error"]["message"]
+ assert message == "HTTP 500: InternalServerError"
+ assert result["error"]["details"] == {"exception_type": "InternalServerError", "status_code": 500}
+ for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):
+ assert leaked not in message.lower()
+
+
@pytest.mark.asyncio
async def test_skyvern_press_key_direct_failure_reports_element_state(monkeypatch: pytest.MonkeyPatch) -> None:
probe_locator = make_probe_locator(count=1, visible=False)
@@ -791,6 +811,16 @@ async def test_skyvern_click_native_option_selector_selects_by_index(monkeypatch
assert result["data"]["sdk_equivalent"] == 'await page.select_option("#region", index=3)'
+def test_exception_message_suppresses_body_outside_4xx() -> None:
+ class NonErrorStatusException(Exception):
+ status_code = 200
+ body = {"detail": "should not be surfaced"}
+
+ message = mcp_browser._exception_message(NonErrorStatusException())
+
+ assert "should not be surfaced" not in message
+
+
@pytest.mark.asyncio
async def test_skyvern_type_ai_error_surfaces_http_status_and_body(monkeypatch: pytest.MonkeyPatch) -> None:
# 4xx bodies are the API's intended client-facing feedback — surfaced for self-correction.
@@ -887,6 +917,58 @@ async def test_skyvern_act_sdk_error_does_not_leak_headers(monkeypatch: pytest.M
assert leaked not in message.lower()
+async def test_skyvern_drag_ai_error_does_not_leak_headers(monkeypatch: pytest.MonkeyPatch) -> None:
+ _action_page(monkeypatch)
+ monkeypatch.setattr(
+ mcp_browser,
+ "do_act",
+ AsyncMock(
+ side_effect=InternalServerError(
+ body={"error": "Unexpected error: sk-BODY-SECRET"},
+ headers={"authorization": "Bearer sk-SECRET"},
+ )
+ ),
+ )
+
+ result = await mcp_browser.skyvern_drag(source_intent="the card", target_intent="the trash bin")
+
+ assert result["ok"] is False
+ message = result["error"]["message"]
+ assert message == "HTTP 500: InternalServerError"
+ assert result["error"]["details"] == {"exception_type": "InternalServerError", "status_code": 500}
+ for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):
+ assert leaked not in message.lower()
+
+
+async def test_skyvern_wait_intent_error_does_not_leak_headers(monkeypatch: pytest.MonkeyPatch) -> None:
+ _action_page(
+ monkeypatch,
+ validate=AsyncMock(
+ side_effect=InternalServerError(
+ body={"error": "Unexpected error: sk-BODY-SECRET"},
+ headers={"authorization": "Bearer sk-SECRET"},
+ )
+ ),
+ wait_for_timeout=make_real_wait_for_timeout(),
+ )
+
+ result = await mcp_browser.skyvern_wait(
+ intent="the spinner disappears",
+ timeout=1000,
+ poll_interval_ms=500,
+ )
+
+ assert result["ok"] is False
+ assert result["error"]["code"] == mcp_browser.ErrorCode.SDK_ERROR
+ message = result["error"]["message"]
+ assert message == "HTTP 500: InternalServerError"
+ assert result["error"]["details"] == {"exception_type": "InternalServerError", "status_code": 500}
+ for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):
+ assert leaked not in message.lower()
+
+
@pytest.mark.asyncio
async def test_skyvern_type_ai_error_uses_exception_type_for_empty_message(
monkeypatch: pytest.MonkeyPatch,
diff --git a/tests/unit/test_mcp_observe_execute.py b/tests/unit/test_mcp_observe_execute.py
index 10d1c5f0e..eb4b1ef41 100644
--- a/tests/unit/test_mcp_observe_execute.py
+++ b/tests/unit/test_mcp_observe_execute.py
@@ -24,7 +24,8 @@ from skyvern.cli.core.result import BrowserContext
from skyvern.cli.core.session_manager import scoped_session
from skyvern.cli.mcp_tools import browser as mcp_browser
from skyvern.cli.mcp_tools import tabs as mcp_tabs
-from tests.unit._mcp_browser_fakes import make_session_state
+from skyvern.client.errors import InternalServerError
+from tests.unit._mcp_browser_fakes import make_real_wait_for_timeout, make_session_state
# ---------------------------------------------------------------------------
# Helpers
@@ -1472,6 +1473,40 @@ class TestSkyvernExecuteMCP:
assert step_error["details"]["element_state"] == "hidden"
assert step_error["details"]["selector"] == "#field"
+ @pytest.mark.asyncio
+ async def test_execute_wait_error_does_not_leak_headers(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ page = _make_page()
+ page.validate = AsyncMock(
+ side_effect=InternalServerError(
+ body={"error": "Unexpected error: sk-BODY-SECRET"},
+ headers={"authorization": "Bearer sk-SECRET"},
+ )
+ )
+ page.wait_for_timeout = make_real_wait_for_timeout()
+ ctx = BrowserContext(mode="local")
+ monkeypatch.setattr(mcp_browser, "get_page", AsyncMock(return_value=(page, ctx)))
+
+ result = await mcp_browser.skyvern_execute(
+ steps=[
+ {
+ "tool": "wait",
+ "params": {
+ "intent": "the spinner disappears",
+ "timeout": 1000,
+ "poll_interval_ms": 500,
+ },
+ }
+ ]
+ )
+
+ assert result["ok"] is False
+ step_error = result["data"]["results"][0]["error"]
+ assert step_error["message"] == "HTTP 500: InternalServerError"
+ assert step_error["details"] == {"exception_type": "InternalServerError", "status_code": 500}
+ message = step_error["message"].lower()
+ for leaked in ("authorization", "bearer", "sk-secret", "sk-body-secret", "headers", "body"):
+ assert leaked not in message
+
@pytest.mark.asyncio
async def test_execute_ref_to_unnamed_duplicate_uses_nth(self, monkeypatch: pytest.MonkeyPatch) -> None:
"""SKY-9701: clicking ref=e2 on a page with 4 unnamed comboboxes resolves to the 3rd, not the 1st."""