Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: Add MCP tool call performance telemetry (#5688)
Marc Kelechavab5d7bcac40d9
diff --git a/skyvern/cli/mcp_tools/telemetry.py b/skyvern/cli/mcp_tools/telemetry.pyindex 0a86b7bad..05fbe1bff 100644--- a/skyvern/cli/mcp_tools/telemetry.py+++ b/skyvern/cli/mcp_tools/telemetry.py@@ -1,5 +1,6 @@from __future__ import annotations+import timefrom contextlib import suppressfrom dataclasses import dataclassfrom typing import Any, Literal@@ -76,6 +77,13 @@ def _resolve_client_id(context: MiddlewareContext[Any]) -> str | None:return None+def _content_text_bytes(content_block: Any) -> int:+ text = getattr(content_block, "text", None)+ if not isinstance(text, str):+ return 0+ return len(text.encode("utf-8"))++def _capture_mcp_event(event_name: str,*,@@ -85,6 +93,8 @@ def _capture_mcp_event(tool_name: str | None = None,prompt_name: str | None = None,error: Exception | None = None,+ duration_ms: float | None = None,+ response_bytes: int | None = None,) -> None:request = _resolve_http_request()organization_id = _resolve_organization_id(request)@@ -112,6 +122,10 @@ def _capture_mcp_event(data["prompt"] = prompt_nameif error is not None:data["error_type"] = type(error).__name__+ if duration_ms is not None:+ data["duration_ms"] = duration_ms+ if response_bytes is not None:+ data["response_bytes"] = response_bytesanalytics.capture(event_name,@@ -160,26 +174,36 @@ class MCPTelemetryMiddleware(Middleware):call_next: CallNext[Any, Any],) -> Any:tool_name = getattr(context.message, "name", None)+ start = time.perf_counter()try:result = await call_next(context)except Exception as exc:+ duration_ms = (time.perf_counter() - start) * 1000+ # Exceptions do not produce MCP content, so response_bytes is only emitted for returned results.+ with suppress(Exception):+ _capture_mcp_event(+ "mcp_tool_call",+ operation="tools/call",+ context=context,+ ok=False,+ tool_name=tool_name,+ error=exc,+ duration_ms=duration_ms,+ )+ raise++ duration_ms = (time.perf_counter() - start) * 1000+ response_bytes = sum(_content_text_bytes(content) for content in (getattr(result, "content", None) or []))+ with suppress(Exception):_capture_mcp_event("mcp_tool_call",operation="tools/call",context=context,- ok=False,+ ok=_resolve_tool_call_ok(result),tool_name=tool_name,- error=exc,+ duration_ms=duration_ms,+ response_bytes=response_bytes,)- raise-- _capture_mcp_event(- "mcp_tool_call",- operation="tools/call",- context=context,- ok=_resolve_tool_call_ok(result),- tool_name=tool_name,- )return resultasync def on_list_tools(diff --git a/tests/unit/test_mcp_telemetry.py b/tests/unit/test_mcp_telemetry.pyindex 74eb11080..5f1c04a0b 100644--- a/tests/unit/test_mcp_telemetry.py+++ b/tests/unit/test_mcp_telemetry.py@@ -2,7 +2,7 @@ from __future__ import annotationsfrom collections.abc import Iteratorfrom types import SimpleNamespace-from unittest.mock import patch+from unittest.mock import Mock, patchimport pytestfrom fastmcp import Client@@ -18,6 +18,15 @@ from skyvern.cli.mcp_tools.telemetry import ()+def _expected_response_bytes(result: object) -> int:+ # Mirrors production telemetry semantics: count UTF-8 bytes for text content blocks only.+ return sum(+ len(content.text.encode("utf-8"))+ for content in (getattr(result, "content", None) or [])+ if isinstance(getattr(content, "text", None), str)+ )++@pytest.fixture(autouse=True)def _reset_runtime() -> Iterator[None]:reset_mcp_telemetry_runtime()@@ -55,12 +64,63 @@ async def test_mcp_tool_call_emits_telemetry() -> None:assert payload["ok"] is Trueassert payload["runtime_mode"] == "local_cli"assert payload["transport"] == "stdio"+ assert isinstance(payload["duration_ms"], float)+ assert payload["duration_ms"] >= 0+ assert payload["response_bytes"] == _expected_response_bytes(result)assert payload["distinct_id_source"] == "analytics_id"assert distinct_id == analytics.settings.ANALYTICS_IDassert api_key == analytics.settings.MCP_POSTHOG_PROJECT_API_KEYassert host == analytics.settings.MCP_POSTHOG_PROJECT_HOST+async def test_mcp_tool_call_records_text_response_bytes() -> None:+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []++ def fake_capture(+ event: str,+ data: dict | None = None,+ distinct_id: str | None = None,+ api_key: str | None = None,+ host: str | None = None,+ ) -> None:+ events.append((event, data, distinct_id, api_key, host))++ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)+ response = SimpleNamespace(+ is_error=False,+ data={"ok": True},+ content=[SimpleNamespace(text="abc"), SimpleNamespace(text="\u00e9"), SimpleNamespace(data="ignored")],+ )++ async def call_next(_context: MiddlewareContext[object]) -> object:+ return response++ with patch.object(analytics, "capture", side_effect=fake_capture):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert len(events) == 1+ _, payload, _, _, _ = events[0]+ assert payload is not None+ assert payload["response_bytes"] == 5+ assert "abc" not in payload.values()+ assert "ignored" not in payload.values()+++async def test_mcp_tool_call_returns_result_when_success_telemetry_fails() -> None:+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)+ response = SimpleNamespace(is_error=False, data={"ok": True}, content=[SimpleNamespace(text="ok")])++ async def call_next(_context: MiddlewareContext[object]) -> object:+ return response++ with patch.object(analytics, "capture", side_effect=RuntimeError("telemetry down")):+ result = await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert result is response++@pytest.mark.asyncioasync def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []@@ -75,9 +135,10 @@ async def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:events.append((event, data, distinct_id, api_key, host))context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)+ response = SimpleNamespace(is_error=True, data={"ok": False}, content=[SimpleNamespace(text="bad")])async def call_next(_context: MiddlewareContext[object]) -> object:- return SimpleNamespace(is_error=True, data={"ok": False})+ return responsewith patch.object(analytics, "capture", side_effect=fake_capture):await MCPTelemetryMiddleware().on_call_tool(context, call_next)@@ -88,6 +149,10 @@ async def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:assert payload is not Noneassert payload["ok"] is Falseassert payload["tool"] == "skyvern_click"+ assert payload["response_bytes"] == 3+ assert isinstance(payload["duration_ms"], float)+ assert "error_type" not in payload+ assert "error_message" not in payload@pytest.mark.asyncio@@ -120,9 +185,26 @@ async def test_mcp_tool_call_exception_omits_error_message() -> None:assert payload is not Noneassert payload["ok"] is Falseassert payload["error_type"] == "ValueError"+ assert isinstance(payload["duration_ms"], float)+ assert payload["duration_ms"] >= 0+ assert "response_bytes" not in payloadassert "error_message" not in payload+async def test_mcp_tool_call_exception_preserves_original_error_when_telemetry_fails() -> None:+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)++ async def call_next(_context: MiddlewareContext[object]) -> object:+ raise ValueError("original tool error")++ with (+ patch.object(analytics, "capture", side_effect=RuntimeError("telemetry down")),+ pytest.raises(ValueError, match="original tool error"),+ ):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++@pytest.mark.asyncioasync def test_list_tools_emits_protocol_request_telemetry() -> None:events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []@@ -179,10 +261,12 @@ async def test_http_request_uses_organization_id_for_distinct_id() -> None:method="POST",)context = MiddlewareContext(message=SimpleNamespace(name="skyvern_block_schema"), fastmcp_context=None)+ response = SimpleNamespace(is_error=False, data={"ok": True}, content=[SimpleNamespace(text="abc")])async def call_next(_context: MiddlewareContext[object]) -> object:- return object()+ return response+ configure_mcp_telemetry_runtime(server_mode="cloud_hosted", transport="streamable-http")with (patch("skyvern.cli.mcp_tools.telemetry.get_http_request", return_value=request),patch.object(analytics, "capture", side_effect=fake_capture),@@ -196,6 +280,29 @@ async def test_http_request_uses_organization_id_for_distinct_id() -> None:assert payload["organization_id"] == "o_test123"assert payload["distinct_id_source"] == "organization_id"assert payload["request_path"] == "/mcp"+ assert payload["runtime_mode"] == "cloud_hosted"+ assert payload["transport"] == "streamable-http"+ assert isinstance(payload["duration_ms"], float)+ assert payload["response_bytes"] == 3assert distinct_id == "org:o_test123"assert api_key == analytics.settings.MCP_POSTHOG_PROJECT_API_KEYassert host == analytics.settings.MCP_POSTHOG_PROJECT_HOST+++async def test_mcp_tool_call_respects_global_telemetry_opt_out() -> None:+ fake_capture = Mock()+ fake_client = SimpleNamespace(capture=fake_capture)+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)+ response = SimpleNamespace(is_error=False, data={"ok": True}, content=[SimpleNamespace(text="abc")])++ async def call_next(_context: MiddlewareContext[object]) -> object:+ return response++ with (+ patch.object(analytics.settings, "SKYVERN_TELEMETRY", False),+ patch.object(analytics, "_resolve_posthog_client", return_value=fake_client),+ ):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ fake_capture.assert_not_called()