Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Reject malformed MCP tool arguments with a clear error (SKY-12666) (#7574)
Aaron Perez892e2a010299
diff --git a/skyvern/cli/mcp_tools/__init__.py b/skyvern/cli/mcp_tools/__init__.pyindex ffc9ac560..30e554342 100644--- a/skyvern/cli/mcp_tools/__init__.py+++ b/skyvern/cli/mcp_tools/__init__.py@@ -184,6 +184,17 @@ def _add_telemetry_middleware() -> None:mcp.add_middleware(MCPTelemetryMiddleware())+def _add_argument_validation_middleware() -> None:+ if _FASTMCP_IMPORT_ERROR is not None:+ return+ from .argument_validation import MCPArgumentValidationMiddleware # noqa: PLC0415++ # Registered after telemetry so telemetry stays the outermost wrapper and+ # still records the call; this one is innermost and rejects malformed+ # arguments before dispatch runs (and logs a raw pydantic error).+ mcp.add_middleware(MCPArgumentValidationMiddleware())++# -- Tool annotation factories --# Every tool registered with FastMCP must carry a human-readable `title`# (required by the Claude Connectors Directory). We build per-tool@@ -366,6 +377,7 @@ Use skyvern_click's resolved_selector response to get xpaths for production scri""",)_add_telemetry_middleware()+_add_argument_validation_middleware()# -- Browser session management --mcp.tool(tags={"session"}, annotations=_mut("Create Browser Session"))(skyvern_browser_session_create)diff --git a/skyvern/cli/mcp_tools/argument_validation.py b/skyvern/cli/mcp_tools/argument_validation.pynew file mode 100644index 000000000..0b56d0528--- /dev/null+++ b/skyvern/cli/mcp_tools/argument_validation.py@@ -0,0 +1,132 @@+"""Pre-dispatch validation of MCP tool-call arguments.++FastMCP validates tool arguments with pydantic *inside* the tool-dispatch core+(``FunctionTool.run`` -> ``type_adapter.validate_python``). When a caller sends+argument names/shapes that don't match a tool's signature, that layer raises a+raw ``pydantic.ValidationError`` which FastMCP logs via+``logger.exception("Error validating tool ...")`` before returning an opaque+failure the calling model cannot act on. The logged exception surfaces in error+tracking as a recurring signature, and the model keeps re-sending the wrong+shape because the failure text is a wall of pydantic internals.++This middleware checks the argument keys against the tool's published input+schema *before* dispatch and short-circuits with a structured error when they+don't match. Because dispatch is never reached, no raw validation error is+logged, and the model receives a clear message naming the unsupported and+expected arguments (the same ``make_result``/``make_error`` envelope every tool+uses). Type-level mismatches with otherwise-valid keys still fall through to+pydantic — this only owns the unsupported-/missing-argument contract. A call+with no ``arguments`` at all also falls through: pydantic handles it, rejecting+only when the tool has required parameters.+"""++from __future__ import annotations++import json+from typing import Any++import structlog+from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext+from fastmcp.tools.tool import ToolResult+from mcp.types import TextContent++from skyvern.cli.core.result import ErrorCode, make_error, make_result++LOG = structlog.get_logger(__name__)+++def _argument_contract(tool: Any) -> tuple[set[str], set[str]] | None:+ """Return ``(allowed, required)`` argument names from a tool's input schema.++ Returns ``None`` when the schema is missing or permits arbitrary properties+ (``additionalProperties``), in which case unsupported-key checks don't apply.+ """+ parameters = getattr(tool, "parameters", None)+ if not isinstance(parameters, dict):+ return None+ # Only an explicit `additionalProperties: false` closes the argument set. A+ # permissive schema ({} or true) allows arbitrary keys, so don't flag extras.+ if parameters.get("additionalProperties", False) is not False:+ return None+ properties = parameters.get("properties")+ if not isinstance(properties, dict):+ return None+ allowed = {str(key) for key in properties}+ required = {str(key) for key in parameters.get("required", []) if isinstance(key, str)}+ return allowed, required+++class MCPArgumentValidationMiddleware(Middleware):+ async def on_call_tool(+ self,+ context: MiddlewareContext[Any],+ call_next: CallNext[Any, Any],+ ) -> Any:+ rejection = await self._reject_bad_arguments(context)+ if rejection is not None:+ return rejection+ return await call_next(context)++ async def _reject_bad_arguments(self, context: MiddlewareContext[Any]) -> ToolResult | None:+ tool_name = getattr(context.message, "name", None)+ arguments = getattr(context.message, "arguments", None)+ if not tool_name or not isinstance(arguments, dict):+ return None++ fastmcp_context = context.fastmcp_context+ if fastmcp_context is None:+ return None+ try:+ tool = await fastmcp_context.fastmcp.get_tool(tool_name)+ except Exception:+ LOG.debug("mcp_argument_validation_skipped_tool_lookup_failed", tool=tool_name, exc_info=True)+ return None+ if tool is None:+ return None++ contract = _argument_contract(tool)+ if contract is None:+ return None+ allowed, required = contract++ provided = {str(key) for key in arguments}+ unsupported = sorted(provided - allowed)+ missing = sorted(required - provided)+ if not unsupported and not missing:+ return None+ return _rejection_result(tool_name, sorted(allowed), unsupported, missing)+++def _rejection_result(+ tool_name: str,+ expected: list[str],+ unsupported: list[str],+ missing: list[str],+) -> ToolResult:+ problems: list[str] = []+ if unsupported:+ problems.append(f"unsupported argument(s): {', '.join(unsupported)}")+ if missing:+ problems.append(f"missing required argument(s): {', '.join(missing)}")+ message = f"{tool_name} was called with {'; '.join(problems)}."+ expected_str = ", ".join(expected) if expected else "(none)"++ payload = make_result(+ tool_name,+ ok=False,+ error=make_error(+ ErrorCode.INVALID_INPUT,+ message,+ f"Call {tool_name} using only its documented parameters: {expected_str}.",+ details={+ "unsupported_arguments": unsupported,+ "missing_required_arguments": missing,+ "expected_arguments": expected,+ },+ ),+ )+ text = json.dumps(payload, ensure_ascii=False, default=str)+ return ToolResult(content=[TextContent(type="text", text=text)], structured_content=payload)+++__all__ = ["MCPArgumentValidationMiddleware"]diff --git a/skyvern/cli/mcp_tools/telemetry.py b/skyvern/cli/mcp_tools/telemetry.pyindex 19d64b31f..4cf717413 100644--- a/skyvern/cli/mcp_tools/telemetry.py+++ b/skyvern/cli/mcp_tools/telemetry.py@@ -179,17 +179,25 @@ def _capture_mcp_event()+def _extract_ok(payload: Any) -> bool | None:+ if isinstance(payload, dict):+ candidate = payload.get("ok")+ if isinstance(candidate, bool):+ return candidate+ return None++def _resolve_tool_call_ok(result: Any) -> bool:is_error = getattr(result, "is_error", None)if not isinstance(is_error, bool):is_error = False- data = getattr(result, "data", None)- result_ok = None- if isinstance(data, dict):- candidate = data.get("ok")- if isinstance(candidate, bool):- result_ok = candidate+ # A FastMCP ToolResult exposes the tool's dict via `structured_content`, not+ # `data`, so a tool (or the argument-validation middleware) reporting failure+ # with `ok=False` must be read there too, or the failure is miscounted as ok.+ result_ok = _extract_ok(getattr(result, "data", None))+ if result_ok is None:+ result_ok = _extract_ok(getattr(result, "structured_content", None))if result_ok is None:return not is_errordiff --git a/tests/unit/test_mcp_argument_validation.py b/tests/unit/test_mcp_argument_validation.pynew file mode 100644index 000000000..ca83db2ea--- /dev/null+++ b/tests/unit/test_mcp_argument_validation.py@@ -0,0 +1,72 @@+"""The argument-validation middleware rejects malformed tool calls with a clear,+structured error instead of letting FastMCP raise a raw pydantic ValidationError.++Each case mirrors a real production error signature where a caller sent argument+names that don't match the tool's contract.+"""++from __future__ import annotations++import pytest++from skyvern.cli.mcp_tools import mcp+++def _structured(result: object) -> dict:+ payload = getattr(result, "structured_content", None)+ assert isinstance(payload, dict), f"expected structured content dict, got {result!r}"+ return payload+++async def _call(tool_name: str, arguments: dict) -> dict:+ result = await mcp.call_tool(tool_name, arguments)+ return _structured(result)+++ ("tool_name", "arguments", "unsupported", "missing"),+ [+ ("skyvern_get_errors", {"run_id": "wr_1"}, ["run_id"], []),+ ("skyvern_get_errors", {"workflow_run_id": "wr_1"}, ["workflow_run_id"], []),+ ("skyvern_wait", {"seconds": "3", "reason": "login"}, ["reason", "seconds"], []),+ ("skyvern_execute", {"tools": [{"tool": "navigate"}]}, ["tools"], ["steps"]),+ ("skyvern_script_get_code", {"workflow_id": "wpid_1"}, ["workflow_id"], ["script_id"]),+ (+ "skyvern_workflow_run",+ {"workflow_id": "wpid_1", "ai_fallback": True, "timeout": 2700},+ ["ai_fallback", "timeout"],+ [],+ ),+ ],+)+async def test_bad_arguments_rejected_with_structured_error(+ tool_name: str, arguments: dict, unsupported: list[str], missing: list[str]+) -> None:+ payload = await _call(tool_name, arguments)++ assert payload["ok"] is False+ error = payload["error"]+ assert error["code"] == "INVALID_INPUT"+ details = error["details"]+ assert details["unsupported_arguments"] == unsupported+ assert details["missing_required_arguments"] == missing+ # The rejection names the accepted arguments so the model can self-correct.+ assert details["expected_arguments"]+ for bad in unsupported:+ assert bad not in details["expected_arguments"]+++async def test_valid_argument_shape_is_not_blocked() -> None:+ """A validly-shaped call is not short-circuited by the middleware.++ ``skyvern_get_errors`` accepts ``text``; with no browser session it fails+ downstream, but with a tool-owned error (not the middleware's INVALID_INPUT+ unsupported-argument rejection), proving the pre-check let it through.+ """+ payload = await _call("skyvern_get_errors", {"text": "boom"})++ error = payload.get("error")+ if error is not None:+ assert "unsupported_arguments" not in error.get("details", {})diff --git a/tests/unit/test_mcp_telemetry.py b/tests/unit/test_mcp_telemetry.pyindex 797f41350..5cae890d8 100644--- a/tests/unit/test_mcp_telemetry.py+++ b/tests/unit/test_mcp_telemetry.py@@ -155,6 +155,38 @@ async def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:assert "error_message" not in payload+async def test_mcp_tool_call_reads_ok_from_structured_content() -> None:+ """A ToolResult reporting failure via structured_content (no `data`) is recorded ok=False.++ The argument-validation middleware short-circuits with exactly this shape.+ """+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []++ def fake_capture(+ event: str,+ data: dict | None = None,+ distinct_id: str | None = None,+ api_key: str | None = None,+ host: str | None = None,+ ) -> None:+ events.append((event, data, distinct_id, api_key, host))++ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_get_errors"), fastmcp_context=None)+ response = SimpleNamespace(structured_content={"ok": False, "error": {"code": "INVALID_INPUT"}}, content=[])++ async def call_next(_context: MiddlewareContext[object]) -> object:+ return response++ with patch.object(analytics, "capture", side_effect=fake_capture):+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)++ assert len(events) == 1+ _, payload, _, _, _ = events[0]+ assert payload is not None+ assert payload["ok"] is False++@pytest.mark.asyncioasync def test_mcp_tool_call_exception_omits_error_message() -> None:events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []