Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Fix MCP PostHog telemetry routing (#SKY-8198) (#5027)
Marc Kelechavab845a6740592
diff --git a/skyvern/analytics.py b/skyvern/analytics.pyindex a561db4be..3d977d8c2 100644--- a/skyvern/analytics.py+++ b/skyvern/analytics.py@@ -12,11 +12,16 @@ from skyvern.config import settingsLOG = structlog.get_logger(__name__)-posthog = Posthog(- "phc_bVT2ugnZhMHRWqMvSRHPdeTjaPxQqT3QSsI3r5FlQR5",- host="https://app.posthog.com",- disable_geoip=False,++def _build_posthog_client(api_key: str, host: str) -> Posthog:+ return Posthog(api_key, host=host, disable_geoip=False)+++posthog = _build_posthog_client(+ settings.POSTHOG_PROJECT_API_KEY,+ settings.POSTHOG_PROJECT_HOST,)+_custom_posthog_clients: dict[tuple[str, str], Posthog] = {}DISTINCT_ID = "oss"@@ -41,17 +46,63 @@ def analytics_metadata() -> Dict[str, Any]:}+def dynamic_analytics_metadata() -> Dict[str, Any]:+ metadata: dict[str, Any] = {}+ if settings.ANALYTICS_TEST_ID:+ metadata["analytics_test_id"] = settings.ANALYTICS_TEST_ID+ return metadata+++def reconfigure_posthog_client(+ api_key: str | None = None,+ host: str | None = None,+) -> None:+ global posthog+ posthog = _build_posthog_client(+ api_key or settings.POSTHOG_PROJECT_API_KEY,+ host or settings.POSTHOG_PROJECT_HOST,+ )+++def _resolve_posthog_client(+ api_key: str | None = None,+ host: str | None = None,+) -> Posthog:+ if api_key is None and host is None:+ return posthog++ resolved_api_key = api_key or settings.POSTHOG_PROJECT_API_KEY+ resolved_host = host or settings.POSTHOG_PROJECT_HOST+ cache_key = (resolved_api_key, resolved_host)+ client = _custom_posthog_clients.get(cache_key)+ if client is None:+ client = _build_posthog_client(resolved_api_key, resolved_host)+ _custom_posthog_clients[cache_key] = client+ return client+++def flush(+ api_key: str | None = None,+ host: str | None = None,+) -> None:+ _resolve_posthog_client(api_key=api_key, host=host).flush()++def capture(event: str,data: dict[str, Any] | None = None,+ distinct_id: str | None = None,+ api_key: str | None = None,+ host: str | None = None,) -> None:if not settings.SKYVERN_TELEMETRY:returntry:- distinct_id = settings.ANALYTICS_ID- payload: dict[str, Any] = data or {}- posthog.capture(distinct_id=distinct_id, event=event, properties=payload)+ resolved_distinct_id = distinct_id or settings.ANALYTICS_ID+ payload: dict[str, Any] = {**dynamic_analytics_metadata(), **(data or {})}+ client = _resolve_posthog_client(api_key=api_key, host=host)+ client.capture(distinct_id=resolved_distinct_id, event=event, properties=payload)except Exception:LOG.debug("analytics capture failed", event=event, exc_info=True)diff --git a/skyvern/cli/core/result.py b/skyvern/cli/core/result.pyindex cd62059fb..a81285ee5 100644--- a/skyvern/cli/core/result.py+++ b/skyvern/cli/core/result.py@@ -5,8 +5,6 @@ from dataclasses import dataclass, fieldfrom datetime import datetime, timezonefrom typing import Any-from skyvern import analytics-# Module-level flag: when True, make_result() strips fields that waste AI context# tokens (echoed inputs, sdk_equivalent, browser_context, timing, empty collections).# Set once at MCP server startup; CLI paths leave it False.@@ -94,20 +92,6 @@ def make_result(warnings: list[str] | None = None,error: dict[str, Any] | None = None,) -> dict[str, Any]:- analytics.capture(- "mcp_tool_call",- data={- **analytics.analytics_metadata(),- "tool": action,- "ok": ok,- # "total" is set by Timer.__exit__; None for early-return paths before Timer starts- "timing_ms": (timing_ms or {}).get("total"),- "error_code": error.get("code") if error else None,- "browser_mode": browser_context.mode if browser_context else None,- "session_id": browser_context.session_id if browser_context else None,- },- )-if _concise_responses:result: dict[str, Any] = {"ok": ok}if error:diff --git a/skyvern/cli/mcp_tools/__init__.py b/skyvern/cli/mcp_tools/__init__.pyindex cce39c9c4..7385c9bd8 100644--- a/skyvern/cli/mcp_tools/__init__.py+++ b/skyvern/cli/mcp_tools/__init__.py@@ -41,6 +41,7 @@ from .session import (skyvern_browser_session_get,skyvern_browser_session_list,)+from .telemetry import MCPTelemetryMiddlewarefrom .workflow import (skyvern_workflow_cancel,skyvern_workflow_create,@@ -248,6 +249,7 @@ To get xpaths, use skyvern_click during MCP exploration — its `resolved_selectgives you the xpath the AI resolved to. Then hardcode that xpath with a prompt fallback in your script.""",)+mcp.add_middleware(MCPTelemetryMiddleware())# -- Browser session management --mcp.tool()(skyvern_browser_session_create)diff --git a/skyvern/cli/mcp_tools/telemetry.py b/skyvern/cli/mcp_tools/telemetry.pynew file mode 100644index 000000000..0a86b7bad--- /dev/null+++ b/skyvern/cli/mcp_tools/telemetry.py@@ -0,0 +1,239 @@+from __future__ import annotations++from contextlib import suppress+from dataclasses import dataclass+from typing import Any, Literal++from fastmcp.server.dependencies import get_http_request+from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext++from skyvern import analytics++MCPServerMode = Literal["embedded", "local_cli", "cloud_hosted"]+++@dataclass(frozen=True)+class MCPRuntimeConfig:+ server_mode: MCPServerMode = "embedded"+ transport: str | None = None+++_runtime_config = MCPRuntimeConfig()+++def configure_mcp_telemetry_runtime(server_mode: MCPServerMode, transport: str | None) -> None:+ global _runtime_config+ _runtime_config = MCPRuntimeConfig(server_mode=server_mode, transport=transport)+++def reset_mcp_telemetry_runtime() -> None:+ configure_mcp_telemetry_runtime(server_mode="embedded", transport=None)+++def _resolve_http_request() -> Any | None:+ try:+ return get_http_request()+ except RuntimeError:+ return None+++def _resolve_organization_id(request: Any | None) -> str | None:+ if request is None:+ return None+ return getattr(getattr(request, "state", None), "organization_id", None)+++def _resolve_distinct_id(organization_id: str | None) -> tuple[str, str]:+ if organization_id:+ return f"org:{organization_id}", "organization_id"+ return analytics.settings.ANALYTICS_ID, "analytics_id"+++def _resolve_request_id(context: MiddlewareContext[Any]) -> str | None:+ fastmcp_context = context.fastmcp_context+ if fastmcp_context is None:+ return None+ with suppress(RuntimeError):+ return fastmcp_context.request_id+ return None+++def _resolve_session_id(context: MiddlewareContext[Any]) -> str | None:+ fastmcp_context = context.fastmcp_context+ if fastmcp_context is None:+ return None+ with suppress(RuntimeError):+ return fastmcp_context.session_id+ return None+++def _resolve_client_id(context: MiddlewareContext[Any]) -> str | None:+ fastmcp_context = context.fastmcp_context+ if fastmcp_context is None:+ return None+ with suppress(RuntimeError):+ return fastmcp_context.client_id+ return None+++def _capture_mcp_event(+ event_name: str,+ *,+ operation: str,+ context: MiddlewareContext[Any],+ ok: bool,+ tool_name: str | None = None,+ prompt_name: str | None = None,+ error: Exception | None = None,+) -> None:+ request = _resolve_http_request()+ organization_id = _resolve_organization_id(request)+ distinct_id, distinct_id_source = _resolve_distinct_id(organization_id)++ data: dict[str, Any] = {+ **analytics.analytics_metadata(),+ "operation": operation,+ "ok": ok,+ "runtime_mode": _runtime_config.server_mode,+ "transport": _runtime_config.transport,+ "is_http": request is not None,+ "request_path": str(request.url.path) if request is not None else None,+ "request_method": str(request.method) if request is not None else None,+ "organization_id": organization_id,+ "distinct_id_source": distinct_id_source,+ "request_id": _resolve_request_id(context),+ "session_id": _resolve_session_id(context),+ "client_id": _resolve_client_id(context),+ }++ if tool_name is not None:+ data["tool"] = tool_name+ if prompt_name is not None:+ data["prompt"] = prompt_name+ if error is not None:+ data["error_type"] = type(error).__name__++ analytics.capture(+ event_name,+ data=data,+ distinct_id=distinct_id,+ api_key=analytics.settings.MCP_POSTHOG_PROJECT_API_KEY,+ host=analytics.settings.MCP_POSTHOG_PROJECT_HOST,+ )+++def _resolve_tool_call_ok(result: Any) -> bool:+ is_error = getattr(result, "is_error", None)+ if not isinstance(is_error, bool):+ is_error = False++ data = getattr(result, "data", None)+ result_ok = None+ if isinstance(data, dict):+ candidate = data.get("ok")+ if isinstance(candidate, bool):+ result_ok = candidate++ if result_ok is None:+ return not is_error+ return (not is_error) and result_ok+++class MCPTelemetryMiddleware(Middleware):+ async def on_initialize(+ self,+ context: MiddlewareContext[Any],+ call_next: CallNext[Any, Any],+ ) -> Any:+ try:+ result = await call_next(context)+ except Exception as exc:+ _capture_mcp_event("mcp_request", operation="initialize", context=context, ok=False, error=exc)+ raise++ _capture_mcp_event("mcp_request", operation="initialize", context=context, ok=True)+ return result++ async def on_call_tool(+ self,+ context: MiddlewareContext[Any],+ call_next: CallNext[Any, Any],+ ) -> Any:+ tool_name = getattr(context.message, "name", None)+ try:+ result = await call_next(context)+ except Exception as exc:+ _capture_mcp_event(+ "mcp_tool_call",+ operation="tools/call",+ context=context,+ ok=False,+ tool_name=tool_name,+ error=exc,+ )+ raise++ _capture_mcp_event(+ "mcp_tool_call",+ operation="tools/call",+ context=context,+ ok=_resolve_tool_call_ok(result),+ tool_name=tool_name,+ )+ return result++ async def on_list_tools(+ self,+ context: MiddlewareContext[Any],+ call_next: CallNext[Any, Any],+ ) -> Any:+ try:+ result = await call_next(context)+ except Exception as exc:+ _capture_mcp_event("mcp_request", operation="tools/list", context=context, ok=False, error=exc)+ raise++ _capture_mcp_event("mcp_request", operation="tools/list", context=context, ok=True)+ return result++ async def on_get_prompt(+ self,+ context: MiddlewareContext[Any],+ call_next: CallNext[Any, Any],+ ) -> Any:+ prompt_name = getattr(context.message, "name", None)+ try:+ result = await call_next(context)+ except Exception as exc:+ _capture_mcp_event(+ "mcp_request",+ operation="prompts/get",+ context=context,+ ok=False,+ prompt_name=prompt_name,+ error=exc,+ )+ raise++ _capture_mcp_event(+ "mcp_request",+ operation="prompts/get",+ context=context,+ ok=True,+ prompt_name=prompt_name,+ )+ return result++ async def on_list_prompts(+ self,+ context: MiddlewareContext[Any],+ call_next: CallNext[Any, Any],+ ) -> Any:+ try:+ result = await call_next(context)+ except Exception as exc:+ _capture_mcp_event("mcp_request", operation="prompts/list", context=context, ok=False, error=exc)+ raise++ _capture_mcp_event("mcp_request", operation="prompts/list", context=context, ok=True)+ return resultdiff --git a/skyvern/cli/run_commands.py b/skyvern/cli/run_commands.pyindex 8c9e2f4ef..b62e1bed6 100644--- a/skyvern/cli/run_commands.py+++ b/skyvern/cli/run_commands.py@@ -21,6 +21,7 @@ from skyvern.cli.core.mcp_http_auth import MCPAPIKeyMiddleware, close_auth_dbfrom skyvern.cli.core.result import set_concise_responsesfrom skyvern.cli.core.session_manager import close_current_session, set_stateless_http_modefrom skyvern.cli.mcp_tools import mcp # Uses standalone fastmcp (v2.x)+from skyvern.cli.mcp_tools.telemetry import configure_mcp_telemetry_runtimefrom skyvern.cli.utils import start_servicesfrom skyvern.config import settingsfrom skyvern.forge.sdk.core import skyvern_context@@ -295,6 +296,7 @@ def run_mcp("""Run the MCP server with configurable transport for local or remote hosting."""path = _normalize_mcp_path(path)stateless_http_enabled = transport != "stdio" and stateless_http+ configure_mcp_telemetry_runtime(server_mode="local_cli", transport=transport)# atexit covers signal-based exits (SIGTERM); finally covers normal# mcp.run() completion or unhandled exceptions. Both are needed because# atexit doesn't fire on normal return and finally doesn't fire on signals.diff --git a/skyvern/config.py b/skyvern/config.pyindex de35705af..f4434b938 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -121,6 +121,11 @@ class Settings(BaseSettings):SKYVERN_TELEMETRY: bool = TrueANALYTICS_ID: str = "anonymous"+ ANALYTICS_TEST_ID: str | None = None+ POSTHOG_PROJECT_API_KEY: str = "phc_bVT2ugnZhMHRWqMvSRHPdeTjaPxQqT3QSsI3r5FlQR5"+ POSTHOG_PROJECT_HOST: str = "https://app.posthog.com"+ MCP_POSTHOG_PROJECT_API_KEY: str | None = "phc_m4epBbGS1Hf4NPRFNpR4WQ9Ob6yGy6SLbQckBxp3n0P"+ MCP_POSTHOG_PROJECT_HOST: str = "https://app.posthog.com"# email settingsSMTP_HOST: str = "localhost"diff --git a/skyvern/forge/api_app.py b/skyvern/forge/api_app.pyindex 7396ab778..d88479724 100644--- a/skyvern/forge/api_app.py+++ b/skyvern/forge/api_app.py@@ -91,12 +91,6 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:except Exception:LOG.exception("Failed to execute api app startup event")- # Close browser sessions left active by a previous process- try:- await forge_app.PERSISTENT_SESSIONS_MANAGER.cleanup_stale_sessions()- except Exception:- LOG.exception("Failed to clean up stale browser sessions")-# Start cleanup scheduler if enabledcleanup_task = start_cleanup_scheduler()if cleanup_task:@@ -118,11 +112,6 @@ async def lifespan(fastapi_app: FastAPI) -> AsyncGenerator[None, Any]:# Stop cleanup schedulerawait stop_cleanup_scheduler()- # Close all persistent browser sessions- from skyvern.webeye.default_persistent_sessions_manager import DefaultPersistentSessionsManager-- await DefaultPersistentSessionsManager.close()-if forge_app.api_app_shutdown_event:LOG.info("Calling api app shutdown event")try:diff --git a/skyvern/forge/forge_app.py b/skyvern/forge/forge_app.pyindex 20fdeac59..2563978e1 100644--- a/skyvern/forge/forge_app.py+++ b/skyvern/forge/forge_app.py@@ -206,7 +206,6 @@ def create_forge_app() -> ForgeApp:app.WORKFLOW_SERVICE = WorkflowService()app.AGENT_FUNCTION = AgentFunction()app.PERSISTENT_SESSIONS_MANAGER = DefaultPersistentSessionsManager(database=app.DATABASE)- app.PERSISTENT_SESSIONS_MANAGER.watch_session_pool()app.BROWSER_SESSION_RECORDING_SERVICE = BrowserSessionRecordingService()app.AZURE_CLIENT_FACTORY = RealAzureClientFactory()diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex 5a4ee0e24..277e91223 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -1,5 +1,4 @@import json-import uuidfrom datetime import datetime, timedeltafrom typing import Any, List, Literal, Sequence, overload@@ -5326,7 +5325,6 @@ class AgentDB(BaseAlchemyDB):if persistent_browser_session.completed_at:return PersistentBrowserSession.model_validate(persistent_browser_session)persistent_browser_session.completed_at = datetime.utcnow()- persistent_browser_session.status = "completed"await session.commit()await session.refresh(persistent_browser_session)return PersistentBrowserSession.model_validate(persistent_browser_session)@@ -5341,34 +5339,6 @@ class AgentDB(BaseAlchemyDB):LOG.error("UnexpectedError", exc_info=True)raise- async def archive_browser_session_address(self, session_id: str, organization_id: str) -> None:- """Suffix browser_address with a unique tag so the unique constraint- no longer blocks new sessions that reuse the same local address."""- try:- async with self.Session() as session:- row = (- await session.scalars(- select(PersistentBrowserSessionModel)- .filter_by(persistent_browser_session_id=session_id)- .filter_by(organization_id=organization_id)- .filter_by(deleted_at=None)- )- ).first()-- if not row or not row.browser_address:- return- if "::closed::" in row.browser_address:- return-- row.browser_address = f"{row.browser_address}::closed::{uuid.uuid4().hex}"- await session.commit()- except SQLAlchemyError:- LOG.error("SQLAlchemyError", exc_info=True)- raise- except Exception:- LOG.error("UnexpectedError", exc_info=True)- raise-async def get_all_active_persistent_browser_sessions(self) -> List[PersistentBrowserSessionModel]:"""Get all active persistent browser sessions across all organizations."""try:@@ -5382,21 +5352,6 @@ class AgentDB(BaseAlchemyDB):LOG.error("UnexpectedError", exc_info=True)raise- async def get_uncompleted_persistent_browser_sessions(self) -> List[PersistentBrowserSessionModel]:- """Get all browser sessions that have not been completed or deleted."""- try:- async with self.Session() as session:- result = await session.execute(- select(PersistentBrowserSessionModel).filter_by(deleted_at=None).filter_by(completed_at=None)- )- return result.scalars().all()- except SQLAlchemyError:- LOG.error("SQLAlchemyError", exc_info=True)- raise- except Exception:- LOG.error("UnexpectedError", exc_info=True)- raise-async def create_task_run(self,task_run_type: RunType,diff --git a/skyvern/forge/sdk/routes/streaming/auth.py b/skyvern/forge/sdk/routes/streaming/auth.pyindex ed5a6b966..fcee39bac 100644--- a/skyvern/forge/sdk/routes/streaming/auth.py+++ b/skyvern/forge/sdk/routes/streaming/auth.py@@ -68,3 +68,33 @@ async def auth(apikey: str | None, token: str | None, websocket: WebSocket) -> sreturn Nonereturn organization_id+++# NOTE(jdo:streaming-local-dev): use this instead of the above `auth`+async def _auth(apikey: str | None, token: str | None, websocket: WebSocket) -> str | None:+ """+ Local dev auth: extracts org_id from API key without strict validation.+ Falls back to o_temp123 if no key provided.+ """++ try:+ await websocket.accept()+ except ConnectionClosedOK:+ LOG.info("WebSocket connection closed cleanly.")+ return None++ # Try to extract real org_id from the API key+ if apikey:+ try:+ from jose import jwt++ from skyvern.config import settings++ payload = jwt.decode(apikey, settings.SECRET_KEY, algorithms=["HS256"])+ org_id = payload.get("sub")+ if org_id:+ return org_id+ except Exception:+ LOG.warning("Local auth: failed to decode API key, falling back to o_temp123")++ return "o_temp123"diff --git a/skyvern/forge/sdk/routes/streaming/channels/message.py b/skyvern/forge/sdk/routes/streaming/channels/message.pyindex 27691e8bf..8fa73314a 100644--- a/skyvern/forge/sdk/routes/streaming/channels/message.py+++ b/skyvern/forge/sdk/routes/streaming/channels/message.py@@ -222,7 +222,7 @@ class MessageChannel:except Exception:pass- del_message_channel(self.client_id, expected=self)+ del_message_channel(self.client_id)return selfdiff --git a/skyvern/forge/sdk/routes/streaming/channels/vnc.py b/skyvern/forge/sdk/routes/streaming/channels/vnc.pyindex 191c836e4..8d788fd7d 100644--- a/skyvern/forge/sdk/routes/streaming/channels/vnc.py+++ b/skyvern/forge/sdk/routes/streaming/channels/vnc.py@@ -221,7 +221,7 @@ class VncChannel:except Exception:pass- del_vnc_channel(self.client_id, expected=self)+ del_vnc_channel(self.client_id)return selfdiff --git a/skyvern/forge/sdk/routes/streaming/messages.py b/skyvern/forge/sdk/routes/streaming/messages.pyindex 62d366a32..3a8909747 100644--- a/skyvern/forge/sdk/routes/streaming/messages.py+++ b/skyvern/forge/sdk/routes/streaming/messages.py@@ -5,8 +5,10 @@ Provides WS endpoints for streaming messages to/from our frontend application.import structlogfrom fastapi import WebSocket+from skyvern.config import settingsfrom skyvern.forge.sdk.routes.routers import base_router, legacy_base_router-from skyvern.forge.sdk.routes.streaming.auth import auth+from skyvern.forge.sdk.routes.streaming.auth import _auth as local_auth+from skyvern.forge.sdk.routes.streaming.auth import auth as real_authfrom skyvern.forge.sdk.routes.streaming.channels.message import (Loops,MessageChannel,@@ -60,6 +62,7 @@ async def messages(client_id: str | None = None,token: str | None = None,) -> None:+ auth = local_auth if settings.ENV == "local" else real_authorganization_id = await auth(apikey=apikey, token=token, websocket=websocket)if not organization_id:diff --git a/skyvern/forge/sdk/routes/streaming/registries.py b/skyvern/forge/sdk/routes/streaming/registries.pyindex 12143a893..689f0c6f2 100644--- a/skyvern/forge/sdk/routes/streaming/registries.py+++ b/skyvern/forge/sdk/routes/streaming/registries.py@@ -35,22 +35,15 @@ def add_vnc_channel(vnc_channel: VncChannel) -> None:vnc_channels[vnc_channel.client_id] = vnc_channel-def get_vnc_channel(client_id: str) -> VncChannel | None:- return vnc_channels.get(client_id)+def get_vnc_channel(client_id: str) -> t.Union[VncChannel, None]:+ return vnc_channels.get(client_id, None)-def del_vnc_channel(client_id: str, *, expected: VncChannel | None = None) -> None:- candidate = vnc_channels.get(client_id)-- if candidate is None:- return-- # Prevent stale channel shutdown from deleting a newer channel that reused- # the same client_id during route transitions/reconnects.- if expected is not None and candidate is not expected:- return-- del vnc_channels[client_id]+def del_vnc_channel(client_id: str) -> None:+ try:+ del vnc_channels[client_id]+ except KeyError:+ pass# a registry for message channels, keyed by `client_id`@@ -61,32 +54,25 @@ def add_message_channel(message_channel: MessageChannel) -> None:message_channels[message_channel.client_id] = message_channel-def get_message_channel(client_id: str) -> MessageChannel | None:- candidate = message_channels.get(client_id)-- if candidate is None:- return None+def get_message_channel(client_id: str) -> t.Union[MessageChannel, None]:+ candidate = message_channels.get(client_id, None)- if candidate.is_open:+ if candidate and candidate.is_open:return candidate- LOG.info(- "MessageChannel: message channel is not open; deleting it",- client_id=candidate.client_id,- )- del_message_channel(candidate.client_id, expected=candidate)- return None-+ if candidate:+ LOG.info(+ "MessageChannel: message channel is not open; deleting it",+ client_id=candidate.client_id,+ )-def del_message_channel(client_id: str, *, expected: MessageChannel | None = None) -> None:- candidate = message_channels.get(client_id)+ del_message_channel(candidate.client_id)- if candidate is None:- return+ return None- # Prevent stale channel shutdown from deleting a newer channel that reused- # the same client_id during route transitions/reconnects.- if expected is not None and candidate is not expected:- return- del message_channels[client_id]+def del_message_channel(client_id: str) -> None:+ try:+ del message_channels[client_id]+ except KeyError:+ passdiff --git a/skyvern/forge/sdk/routes/streaming/verify.py b/skyvern/forge/sdk/routes/streaming/verify.pyindex 354ace714..1add0d399 100644--- a/skyvern/forge/sdk/routes/streaming/verify.py+++ b/skyvern/forge/sdk/routes/streaming/verify.py@@ -21,6 +21,7 @@ from datetime import datetime, timedeltaimport structlog+from skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.schemas.persistent_browser_sessions import AddressablePersistentBrowserSession, is_final_statusfrom skyvern.forge.sdk.schemas.tasks import Task, TaskStatus@@ -44,6 +45,18 @@ async def verify_browser_session("""Verify the browser session exists, and is usable."""+ if settings.ENV == "local":+ dummy_browser_session = AddressablePersistentBrowserSession(+ persistent_browser_session_id=browser_session_id,+ organization_id=organization_id,+ browser_address="0.0.0.0:9223",+ ip_address="localhost",+ created_at=datetime.now(),+ modified_at=datetime.now(),+ )++ return dummy_browser_session+browser_session = await app.PERSISTENT_SESSIONS_MANAGER.get_session(browser_session_id, organization_id)if not browser_session:@@ -171,6 +184,28 @@ async def verify_workflow_run(with it."""+ if settings.ENV == "local":+ dummy_workflow_run = WorkflowRun(+ workflow_id="123",+ workflow_permanent_id="wpid_123",+ workflow_run_id=workflow_run_id,+ organization_id=organization_id,+ status=WorkflowRunStatus.running,+ created_at=datetime.now(),+ modified_at=datetime.now(),+ )++ dummy_browser_session = AddressablePersistentBrowserSession(+ persistent_browser_session_id=workflow_run_id,+ organization_id=organization_id,+ browser_address="0.0.0.0:9223",+ ip_address="localhost",+ created_at=datetime.now(),+ modified_at=datetime.now(),+ )++ return dummy_workflow_run, dummy_browser_session+workflow_run = await app.DATABASE.get_workflow_run(workflow_run_id=workflow_run_id,organization_id=organization_id,diff --git a/skyvern/forge/sdk/routes/streaming/vnc.py b/skyvern/forge/sdk/routes/streaming/vnc.pyindex 512beb820..8d603ab22 100644--- a/skyvern/forge/sdk/routes/streaming/vnc.py+++ b/skyvern/forge/sdk/routes/streaming/vnc.py@@ -13,8 +13,10 @@ import structlogfrom fastapi import WebSocketfrom websockets.exceptions import ConnectionClosedOK+from skyvern.config import settingsfrom skyvern.forge.sdk.routes.routers import base_router, legacy_base_router-from skyvern.forge.sdk.routes.streaming.auth import auth+from skyvern.forge.sdk.routes.streaming.auth import _auth as local_auth+from skyvern.forge.sdk.routes.streaming.auth import auth as real_authfrom skyvern.forge.sdk.routes.streaming.channels.vnc import (Loops,VncChannel,@@ -87,6 +89,7 @@ async def stream(workflow_run_id=workflow_run_id,)+ auth = local_auth if settings.ENV == "local" else real_authorganization_id = await auth(apikey=apikey, token=token, websocket=websocket)if not organization_id:diff --git a/skyvern/webeye/cdp_ports.py b/skyvern/webeye/cdp_ports.pydeleted file mode 100644index 37ded6fc6..000000000--- a/skyvern/webeye/cdp_ports.py+++ /dev/null@@ -1,29 +0,0 @@-"""CDP port allocation for local browser sessions."""--from __future__ import annotations--import socket--_CDP_PORT_RANGE_START = 9223-_CDP_PORT_RANGE_END = 9322-_allocated_ports: set[int] = set()---def _allocate_cdp_port() -> int:- """Find an available port in the CDP port range for a browser session."""- for port in range(_CDP_PORT_RANGE_START, _CDP_PORT_RANGE_END + 1):- if port in _allocated_ports:- continue- try:- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:- s.bind(("127.0.0.1", port))- _allocated_ports.add(port)- return port- except OSError:- pass- raise RuntimeError(f"No available CDP ports in range {_CDP_PORT_RANGE_START}-{_CDP_PORT_RANGE_END}")---def _release_cdp_port(port: int) -> None:- """Return a CDP port to the available pool."""- _allocated_ports.discard(port)diff --git a/skyvern/webeye/default_persistent_sessions_manager.py b/skyvern/webeye/default_persistent_sessions_manager.pyindex 8cd187735..0695feafe 100644--- a/skyvern/webeye/default_persistent_sessions_manager.py+++ b/skyvern/webeye/default_persistent_sessions_manager.py@@ -7,7 +7,6 @@ from pathlib import Pathimport structlogfrom playwright._impl._errors import TargetClosedError-from playwright.async_api import async_playwrightfrom skyvern.config import settingsfrom skyvern.exceptions import BrowserSessionNotRenewable, MissingBrowserAddressError@@ -22,11 +21,8 @@ from skyvern.forge.sdk.schemas.persistent_browser_sessions import (is_final_status,)from skyvern.schemas.runs import ProxyLocation, ProxyLocationInput-from skyvern.webeye.browser_factory import BrowserContextFactoryfrom skyvern.webeye.browser_state import BrowserState-from skyvern.webeye.cdp_ports import _allocate_cdp_port, _release_cdp_portfrom skyvern.webeye.persistent_sessions_manager import PersistentSessionsManager-from skyvern.webeye.real_browser_state import RealBrowserStateLOG = structlog.get_logger()@@ -34,7 +30,6 @@ LOG = structlog.get_logger()@dataclassclass BrowserSession:browser_state: BrowserState- cdp_port: int | None = Noneasync def validate_session_for_renewal(@@ -182,7 +177,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):"""Default (OSS) implementation of PersistentSessionsManager protocol."""instance: DefaultPersistentSessionsManager | None = None- _browser_sessions: dict[str, BrowserSession] = {}+ _browser_sessions: dict[str, BrowserSession] = dict()database: AgentDBdef __new__(cls, database: AgentDB) -> DefaultPersistentSessionsManager:@@ -196,72 +191,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):return cls.instancedef watch_session_pool(self) -> None:- """No-op in OSS: browsers run in-process, no external pool to monitor."""-- async def _launch_browser_for_session(- self,- session_id: str,- organization_id: str,- ) -> None:- """Launch a browser process and register it as a persistent session."""- cdp_port = _allocate_cdp_port()- LOG.info("Launching browser for persistent session", session_id=session_id, cdp_port=cdp_port)-- pw = None- browser_state = None- try:- pw = await async_playwright().start()- browser_context, browser_artifacts, browser_cleanup = await BrowserContextFactory.create_browser_context(- pw,- organization_id=organization_id,- cdp_port=cdp_port,- )-- browser_state = RealBrowserState(- pw=pw,- browser_context=browser_context,- page=None,- browser_artifacts=browser_artifacts,- browser_cleanup=browser_cleanup,- )- await browser_state.get_or_create_page(organization_id=organization_id)-- self._browser_sessions[session_id] = BrowserSession(- browser_state=browser_state,- cdp_port=cdp_port,- )-- browser_address = f"http://127.0.0.1:{cdp_port}"- await self.database.set_persistent_browser_session_browser_address(- browser_session_id=session_id,- browser_address=browser_address,- ip_address="127.0.0.1",- ecs_task_arn=None,- organization_id=organization_id,- )- await self.database.update_persistent_browser_session(- session_id,- organization_id=organization_id,- status=PersistentBrowserSessionStatus.running,- )-- LOG.info("Browser launched for persistent session", session_id=session_id, browser_address=browser_address)- except BaseException:- _release_cdp_port(cdp_port)- # Close whichever resource was successfully created.- # browser_state.close() stops playwright internally, so only fall- # back to pw.stop() when no browser_state was created.- if browser_state is not None:- try:- await browser_state.close()- except Exception:- LOG.warning("Failed to close browser_state during cleanup", exc_info=True)- elif pw is not None:- try:- await pw.stop()- except Exception:- LOG.warning("Failed to stop playwright during cleanup", exc_info=True)- raise+ return Noneasync def begin_session(self,@@ -345,7 +275,7 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):"Creating new browser session",organization_id=organization_id,)- session = await self.database.create_persistent_browser_session(+ return await self.database.create_persistent_browser_session(organization_id=organization_id,runnable_type=runnable_type,runnable_id=runnable_id,@@ -356,27 +286,6 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):browser_profile_id=browser_profile_id,)- try:- await self._launch_browser_for_session(- session_id=session.persistent_browser_session_id,- organization_id=organization_id,- )- # Re-fetch to get updated browser_address/ip_address/started_at- updated = await self.database.get_persistent_browser_session(- session_id=session.persistent_browser_session_id,- organization_id=organization_id,- )- if updated:- return updated- except Exception:- LOG.exception(- "Failed to launch browser for session, session will have no browser",- session_id=session.persistent_browser_session_id,- organization_id=organization_id,- )-- return session-async def occupy_browser_session(self,session_id: str,@@ -462,8 +371,6 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):)self._browser_sessions.pop(browser_session_id, None)- if browser_session.cdp_port is not None:- _release_cdp_port(browser_session.cdp_port)try:await browser_session.browser_state.close()@@ -488,8 +395,6 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):)await self.database.close_persistent_browser_session(browser_session_id, organization_id)- if settings.ENV == "local":- await self.database.archive_browser_session_address(browser_session_id, organization_id)async def close_all_sessions(self, organization_id: str) -> None:"""Close all browser sessions for an organization."""@@ -497,24 +402,6 @@ class DefaultPersistentSessionsManager(PersistentSessionsManager):for browser_session in browser_sessions:await self.close_session(organization_id, browser_session.persistent_browser_session_id)- async def cleanup_stale_sessions(self) -> None:- """Close sessions left active by a previous process."""- if settings.ENV != "local":- return- stale_sessions = await self.database.get_uncompleted_persistent_browser_sessions()- for db_session in stale_sessions:- LOG.info(- "Closing stale browser session from previous run",- session_id=db_session.persistent_browser_session_id,- organization_id=db_session.organization_id,- )- await self.database.close_persistent_browser_session(- db_session.persistent_browser_session_id, db_session.organization_id- )- await self.database.archive_browser_session_address(- db_session.persistent_browser_session_id, db_session.organization_id- )-@classmethodasync def close(cls) -> None:"""Close all browser sessions across all organizations."""diff --git a/skyvern/webeye/persistent_sessions_manager.py b/skyvern/webeye/persistent_sessions_manager.pyindex 05da670bd..3eeea1f05 100644--- a/skyvern/webeye/persistent_sessions_manager.py+++ b/skyvern/webeye/persistent_sessions_manager.py@@ -105,10 +105,6 @@ class PersistentSessionsManager(Protocol):"""Close all browser sessions for an organization."""...- async def cleanup_stale_sessions(self) -> None:- """Clean up sessions left active by a previous process."""- ...-@classmethodasync def close(cls) -> None:"""Close all browser sessions across all organizations."""diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.pynew file mode 100644index 000000000..5f4226e53--- /dev/null+++ b/tests/unit/test_analytics.py@@ -0,0 +1,62 @@+from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import Mock, patch++from skyvern import analytics+++def test_capture_includes_dynamic_test_id() -> None:+ original_test_id = analytics.settings.ANALYTICS_TEST_ID+ analytics.settings.ANALYTICS_TEST_ID = "smoke-test-123"+ fake_capture = Mock()+ fake_client = SimpleNamespace(capture=fake_capture)++ try:+ with patch.object(analytics, "_resolve_posthog_client", return_value=fake_client):+ analytics.capture("mcp_request", data={"ok": True}, distinct_id="distinct-123")+ finally:+ analytics.settings.ANALYTICS_TEST_ID = original_test_id++ fake_capture.assert_called_once_with(+ distinct_id="distinct-123",+ event="mcp_request",+ properties={"analytics_test_id": "smoke-test-123", "ok": True},+ )+++def test_reconfigure_posthog_client_uses_project_settings() -> None:+ original_api_key = analytics.settings.POSTHOG_PROJECT_API_KEY+ original_host = analytics.settings.POSTHOG_PROJECT_HOST++ analytics.settings.POSTHOG_PROJECT_API_KEY = "phc_test_project_key"+ analytics.settings.POSTHOG_PROJECT_HOST = "https://app.posthog.com"++ try:+ analytics.reconfigure_posthog_client()+ assert analytics.posthog.api_key == "phc_test_project_key"+ finally:+ analytics.settings.POSTHOG_PROJECT_API_KEY = original_api_key+ analytics.settings.POSTHOG_PROJECT_HOST = original_host+ analytics.reconfigure_posthog_client()+++def test_capture_can_use_custom_posthog_client() -> None:+ fake_capture = Mock()+ fake_client = SimpleNamespace(capture=fake_capture)++ with patch.object(analytics, "_build_posthog_client", return_value=fake_client):+ analytics._custom_posthog_clients.clear()+ analytics.capture(+ "mcp_tool_call",+ data={"ok": True},+ distinct_id="distinct-123",+ api_key="phc_mcp_project",+ host="https://app.posthog.com",+ )++ fake_capture.assert_called_once_with(+ distinct_id="distinct-123",+ event="mcp_tool_call",+ properties={"ok": True},+ )diff --git a/tests/unit/test_mcp_telemetry.py b/tests/unit/test_mcp_telemetry.pynew file mode 100644index 000000000..74eb11080--- /dev/null+++ b/tests/unit/test_mcp_telemetry.py@@ -0,0 +1,201 @@+from __future__ import annotations++from collections.abc import Iterator+from types import SimpleNamespace+from unittest.mock import patch++import pytest+from fastmcp import Client+from fastmcp.server.middleware import MiddlewareContext++from skyvern import analytics+from skyvern.cli.mcp_tools import mcp+from skyvern.cli.mcp_tools.blocks import skyvern_block_schema+from skyvern.cli.mcp_tools.telemetry import (+ MCPTelemetryMiddleware,+ configure_mcp_telemetry_runtime,+ reset_mcp_telemetry_runtime,+)++[email protected](autouse=True)+def _reset_runtime() -> Iterator[None]:+ reset_mcp_telemetry_runtime()+ yield+ reset_mcp_telemetry_runtime()+++async def test_mcp_tool_call_emits_telemetry() -> None:+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []++ def fake_capture(+ event: str,+ data: dict | None = None,+ distinct_id: str | None = None,+ api_key: str | None = None,+ host: str | None = None,+ ) -> None:+ events.append((event, data, distinct_id, api_key, host))++ configure_mcp_telemetry_runtime(server_mode="local_cli", transport="stdio")++ with patch.object(analytics, "capture", side_effect=fake_capture):+ async with Client(mcp) as client:+ result = await client.call_tool("skyvern_block_schema", {})++ assert result.is_error is False+ tool_events = [event for event in events if event[0] == "mcp_tool_call"]+ assert len(tool_events) == 1++ _, payload, distinct_id, api_key, host = tool_events[0]+ assert payload is not None+ assert payload["operation"] == "tools/call"+ assert payload["tool"] == "skyvern_block_schema"+ assert payload["ok"] is True+ assert payload["runtime_mode"] == "local_cli"+ assert payload["transport"] == "stdio"+ assert payload["distinct_id_source"] == "analytics_id"+ assert distinct_id == analytics.settings.ANALYTICS_ID+ assert api_key == analytics.settings.MCP_POSTHOG_PROJECT_API_KEY+ assert host == analytics.settings.MCP_POSTHOG_PROJECT_HOST+++async def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []++ def fake_capture(+ event: str,+ data: dict | None = None,+ distinct_id: str | None = None,+ api_key: str | None = None,+ host: str | None = None,+ ) -> None:+ events.append((event, data, distinct_id, api_key, host))++ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)++ async def call_next(_context: MiddlewareContext[object]) -> object:+ return SimpleNamespace(is_error=True, data={"ok": False})++ with patch.object(analytics, "capture", side_effect=fake_capture):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert len(events) == 1+ event_name, payload, _, _, _ = events[0]+ assert event_name == "mcp_tool_call"+ assert payload is not None+ assert payload["ok"] is False+ assert payload["tool"] == "skyvern_click"+++async def test_mcp_tool_call_exception_omits_error_message() -> None:+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []++ def fake_capture(+ event: str,+ data: dict | None = None,+ distinct_id: str | None = None,+ api_key: str | None = None,+ host: str | None = None,+ ) -> None:+ events.append((event, data, distinct_id, api_key, host))++ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_click"), fastmcp_context=None)++ async def call_next(_context: MiddlewareContext[object]) -> object:+ raise ValueError("sensitive input should not leave the process")++ with (+ patch.object(analytics, "capture", side_effect=fake_capture),+ pytest.raises(ValueError, match="sensitive input should not leave the process"),+ ):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert len(events) == 1+ event_name, payload, _, _, _ = events[0]+ assert event_name == "mcp_tool_call"+ assert payload is not None+ assert payload["ok"] is False+ assert payload["error_type"] == "ValueError"+ assert "error_message" not in payload+++async def test_list_tools_emits_protocol_request_telemetry() -> None:+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []++ def fake_capture(+ event: str,+ data: dict | None = None,+ distinct_id: str | None = None,+ api_key: str | None = None,+ host: str | None = None,+ ) -> None:+ events.append((event, data, distinct_id, api_key, host))++ with patch.object(analytics, "capture", side_effect=fake_capture):+ async with Client(mcp) as client:+ tools = await client.list_tools()++ assert tools+ assert any(+ event == "mcp_request" and payload and payload["operation"] == "initialize"+ for event, payload, _, _, _ in events+ )+ assert any(+ event == "mcp_request" and payload and payload["operation"] == "tools/list"+ for event, payload, _, _, _ in events+ )+++async def test_direct_tool_invocation_does_not_emit_mcp_telemetry() -> None:+ with patch.object(analytics, "capture") as capture_mock:+ result = await skyvern_block_schema()++ assert result["ok"] is True+ capture_mock.assert_not_called()+++async def test_http_request_uses_organization_id_for_distinct_id() -> None:+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []++ def fake_capture(+ event: str,+ data: dict | None = None,+ distinct_id: str | None = None,+ api_key: str | None = None,+ host: str | None = None,+ ) -> None:+ events.append((event, data, distinct_id, api_key, host))++ request = SimpleNamespace(+ state=SimpleNamespace(organization_id="o_test123"),+ url=SimpleNamespace(path="/mcp"),+ method="POST",+ )+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_block_schema"), fastmcp_context=None)++ async def call_next(_context: MiddlewareContext[object]) -> object:+ return object()++ with (+ patch("skyvern.cli.mcp_tools.telemetry.get_http_request", return_value=request),+ patch.object(analytics, "capture", side_effect=fake_capture),+ ):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert len(events) == 1+ event_name, payload, distinct_id, api_key, host = events[0]+ assert event_name == "mcp_tool_call"+ assert payload is not None+ assert payload["organization_id"] == "o_test123"+ assert payload["distinct_id_source"] == "organization_id"+ assert payload["request_path"] == "/mcp"+ assert distinct_id == "org:o_test123"+ assert api_key == analytics.settings.MCP_POSTHOG_PROJECT_API_KEY+ assert host == analytics.settings.MCP_POSTHOG_PROJECT_HOSTdiff --git a/tests/unit_tests/test_cdp_port_allocator.py b/tests/unit_tests/test_cdp_port_allocator.pydeleted file mode 100644index 93f873468..000000000--- a/tests/unit_tests/test_cdp_port_allocator.py+++ /dev/null@@ -1,76 +0,0 @@-import socket--import pytest--from skyvern.webeye.cdp_ports import (- _CDP_PORT_RANGE_END,- _CDP_PORT_RANGE_START,- _allocate_cdp_port,- _allocated_ports,- _release_cdp_port,-)--[email protected](autouse=True)-def _clean_allocated_ports():- """Ensure the global allocated-ports set is empty before and after each test."""- _allocated_ports.clear()- yield- _allocated_ports.clear()---class TestAllocateCdpPort:- def test_returns_port_in_range(self):- port = _allocate_cdp_port()- assert _CDP_PORT_RANGE_START <= port <= _CDP_PORT_RANGE_END-- def test_port_is_tracked(self):- port = _allocate_cdp_port()- assert port in _allocated_ports-- def test_consecutive_calls_return_different_ports(self):- p1 = _allocate_cdp_port()- p2 = _allocate_cdp_port()- assert p1 != p2-- def test_skips_already_allocated_ports(self):- first = _allocate_cdp_port()- second = _allocate_cdp_port()- assert second != first- assert {first, second}.issubset(_allocated_ports)-- def test_skips_ports_bound_by_other_processes(self):- # Bind the first port in the range so the allocator must skip it.- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)- sock.bind(("127.0.0.1", _CDP_PORT_RANGE_START))- try:- port = _allocate_cdp_port()- assert port != _CDP_PORT_RANGE_START- assert port in _allocated_ports- finally:- sock.close()-- def test_raises_when_range_exhausted(self):- # Mark every port in the range as allocated.- for p in range(_CDP_PORT_RANGE_START, _CDP_PORT_RANGE_END + 1):- _allocated_ports.add(p)-- with pytest.raises(RuntimeError, match="No available CDP ports"):- _allocate_cdp_port()---class TestReleaseCdpPort:- def test_release_removes_from_tracking(self):- port = _allocate_cdp_port()- _release_cdp_port(port)- assert port not in _allocated_ports-- def test_release_allows_reallocation(self):- p1 = _allocate_cdp_port()- _release_cdp_port(p1)- p2 = _allocate_cdp_port()- assert p2 == p1-- def test_release_nonexistent_port_is_noop(self):- _release_cdp_port(99999)- assert 99999 not in _allocated_ports