Skip to content

Automate browser based workflows with AI

Reject malformed MCP tool arguments with a clear error (SKY-12666) (#7574)

Aaron Perez892e2a010299
diff --git a/skyvern/cli/mcp_tools/__init__.py b/skyvern/cli/mcp_tools/__init__.py
index ffc9ac560..30e554342 100644
--- a/skyvern/cli/mcp_tools/__init__.py
+++ b/skyvern/cli/mcp_tools/__init__.py
@@ -184,6 +184,17 @@ def _add_telemetry_middleware() -> None:
mcp.add_middleware(MCPTelemetryMiddleware())
+def _add_argument_validation_middleware() -> None:
+ if _FASTMCP_IMPORT_ERROR is not None:
+ return
+ from .argument_validation import MCPArgumentValidationMiddleware # noqa: PLC0415
+
+ # Registered after telemetry so telemetry stays the outermost wrapper and
+ # still records the call; this one is innermost and rejects malformed
+ # arguments before dispatch runs (and logs a raw pydantic error).
+ mcp.add_middleware(MCPArgumentValidationMiddleware())
+
+
# -- Tool annotation factories --
# Every tool registered with FastMCP must carry a human-readable `title`
# (required by the Claude Connectors Directory). We build per-tool
@@ -366,6 +377,7 @@ Use skyvern_click's resolved_selector response to get xpaths for production scri
""",
)
_add_telemetry_middleware()
+_add_argument_validation_middleware()
# -- Browser session management --
mcp.tool(tags={"session"}, annotations=_mut("Create Browser Session"))(skyvern_browser_session_create)
diff --git a/skyvern/cli/mcp_tools/argument_validation.py b/skyvern/cli/mcp_tools/argument_validation.py
new file mode 100644
index 000000000..0b56d0528
--- /dev/null
+++ b/skyvern/cli/mcp_tools/argument_validation.py
@@ -0,0 +1,132 @@
+"""Pre-dispatch validation of MCP tool-call arguments.
+
+FastMCP validates tool arguments with pydantic *inside* the tool-dispatch core
+(``FunctionTool.run`` -> ``type_adapter.validate_python``). When a caller sends
+argument names/shapes that don't match a tool's signature, that layer raises a
+raw ``pydantic.ValidationError`` which FastMCP logs via
+``logger.exception("Error validating tool ...")`` before returning an opaque
+failure the calling model cannot act on. The logged exception surfaces in error
+tracking as a recurring signature, and the model keeps re-sending the wrong
+shape because the failure text is a wall of pydantic internals.
+
+This middleware checks the argument keys against the tool's published input
+schema *before* dispatch and short-circuits with a structured error when they
+don't match. Because dispatch is never reached, no raw validation error is
+logged, and the model receives a clear message naming the unsupported and
+expected arguments (the same ``make_result``/``make_error`` envelope every tool
+uses). Type-level mismatches with otherwise-valid keys still fall through to
+pydantic — this only owns the unsupported-/missing-argument contract. A call
+with no ``arguments`` at all also falls through: pydantic handles it, rejecting
+only when the tool has required parameters.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import structlog
+from fastmcp.server.middleware import CallNext, Middleware, MiddlewareContext
+from fastmcp.tools.tool import ToolResult
+from mcp.types import TextContent
+
+from skyvern.cli.core.result import ErrorCode, make_error, make_result
+
+LOG = structlog.get_logger(__name__)
+
+
+def _argument_contract(tool: Any) -> tuple[set[str], set[str]] | None:
+ """Return ``(allowed, required)`` argument names from a tool's input schema.
+
+ Returns ``None`` when the schema is missing or permits arbitrary properties
+ (``additionalProperties``), in which case unsupported-key checks don't apply.
+ """
+ parameters = getattr(tool, "parameters", None)
+ if not isinstance(parameters, dict):
+ return None
+ # Only an explicit `additionalProperties: false` closes the argument set. A
+ # permissive schema ({} or true) allows arbitrary keys, so don't flag extras.
+ if parameters.get("additionalProperties", False) is not False:
+ return None
+ properties = parameters.get("properties")
+ if not isinstance(properties, dict):
+ return None
+ allowed = {str(key) for key in properties}
+ required = {str(key) for key in parameters.get("required", []) if isinstance(key, str)}
+ return allowed, required
+
+
+class MCPArgumentValidationMiddleware(Middleware):
+ async def on_call_tool(
+ self,
+ context: MiddlewareContext[Any],
+ call_next: CallNext[Any, Any],
+ ) -> Any:
+ rejection = await self._reject_bad_arguments(context)
+ if rejection is not None:
+ return rejection
+ return await call_next(context)
+
+ async def _reject_bad_arguments(self, context: MiddlewareContext[Any]) -> ToolResult | None:
+ tool_name = getattr(context.message, "name", None)
+ arguments = getattr(context.message, "arguments", None)
+ if not tool_name or not isinstance(arguments, dict):
+ return None
+
+ fastmcp_context = context.fastmcp_context
+ if fastmcp_context is None:
+ return None
+ try:
+ tool = await fastmcp_context.fastmcp.get_tool(tool_name)
+ except Exception:
+ LOG.debug("mcp_argument_validation_skipped_tool_lookup_failed", tool=tool_name, exc_info=True)
+ return None
+ if tool is None:
+ return None
+
+ contract = _argument_contract(tool)
+ if contract is None:
+ return None
+ allowed, required = contract
+
+ provided = {str(key) for key in arguments}
+ unsupported = sorted(provided - allowed)
+ missing = sorted(required - provided)
+ if not unsupported and not missing:
+ return None
+ return _rejection_result(tool_name, sorted(allowed), unsupported, missing)
+
+
+def _rejection_result(
+ tool_name: str,
+ expected: list[str],
+ unsupported: list[str],
+ missing: list[str],
+) -> ToolResult:
+ problems: list[str] = []
+ if unsupported:
+ problems.append(f"unsupported argument(s): {', '.join(unsupported)}")
+ if missing:
+ problems.append(f"missing required argument(s): {', '.join(missing)}")
+ message = f"{tool_name} was called with {'; '.join(problems)}."
+ expected_str = ", ".join(expected) if expected else "(none)"
+
+ payload = make_result(
+ tool_name,
+ ok=False,
+ error=make_error(
+ ErrorCode.INVALID_INPUT,
+ message,
+ f"Call {tool_name} using only its documented parameters: {expected_str}.",
+ details={
+ "unsupported_arguments": unsupported,
+ "missing_required_arguments": missing,
+ "expected_arguments": expected,
+ },
+ ),
+ )
+ text = json.dumps(payload, ensure_ascii=False, default=str)
+ return ToolResult(content=[TextContent(type="text", text=text)], structured_content=payload)
+
+
+__all__ = ["MCPArgumentValidationMiddleware"]
diff --git a/skyvern/cli/mcp_tools/telemetry.py b/skyvern/cli/mcp_tools/telemetry.py
index 19d64b31f..4cf717413 100644
--- a/skyvern/cli/mcp_tools/telemetry.py
+++ b/skyvern/cli/mcp_tools/telemetry.py
@@ -179,17 +179,25 @@ def _capture_mcp_event(
)
+def _extract_ok(payload: Any) -> bool | None:
+ if isinstance(payload, dict):
+ candidate = payload.get("ok")
+ if isinstance(candidate, bool):
+ return candidate
+ return None
+
+
def _resolve_tool_call_ok(result: Any) -> bool:
is_error = getattr(result, "is_error", None)
if not isinstance(is_error, bool):
is_error = False
- data = getattr(result, "data", None)
- result_ok = None
- if isinstance(data, dict):
- candidate = data.get("ok")
- if isinstance(candidate, bool):
- result_ok = candidate
+ # A FastMCP ToolResult exposes the tool's dict via `structured_content`, not
+ # `data`, so a tool (or the argument-validation middleware) reporting failure
+ # with `ok=False` must be read there too, or the failure is miscounted as ok.
+ result_ok = _extract_ok(getattr(result, "data", None))
+ if result_ok is None:
+ result_ok = _extract_ok(getattr(result, "structured_content", None))
if result_ok is None:
return not is_error
diff --git a/tests/unit/test_mcp_argument_validation.py b/tests/unit/test_mcp_argument_validation.py
new file mode 100644
index 000000000..ca83db2ea
--- /dev/null
+++ b/tests/unit/test_mcp_argument_validation.py
@@ -0,0 +1,72 @@
+"""The argument-validation middleware rejects malformed tool calls with a clear,
+structured error instead of letting FastMCP raise a raw pydantic ValidationError.
+
+Each case mirrors a real production error signature where a caller sent argument
+names that don't match the tool's contract.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from skyvern.cli.mcp_tools import mcp
+
+
+def _structured(result: object) -> dict:
+ payload = getattr(result, "structured_content", None)
+ assert isinstance(payload, dict), f"expected structured content dict, got {result!r}"
+ return payload
+
+
+async def _call(tool_name: str, arguments: dict) -> dict:
+ result = await mcp.call_tool(tool_name, arguments)
+ return _structured(result)
+
+
+ ("tool_name", "arguments", "unsupported", "missing"),
+ [
+ ("skyvern_get_errors", {"run_id": "wr_1"}, ["run_id"], []),
+ ("skyvern_get_errors", {"workflow_run_id": "wr_1"}, ["workflow_run_id"], []),
+ ("skyvern_wait", {"seconds": "3", "reason": "login"}, ["reason", "seconds"], []),
+ ("skyvern_execute", {"tools": [{"tool": "navigate"}]}, ["tools"], ["steps"]),
+ ("skyvern_script_get_code", {"workflow_id": "wpid_1"}, ["workflow_id"], ["script_id"]),
+ (
+ "skyvern_workflow_run",
+ {"workflow_id": "wpid_1", "ai_fallback": True, "timeout": 2700},
+ ["ai_fallback", "timeout"],
+ [],
+ ),
+ ],
+)
+async def test_bad_arguments_rejected_with_structured_error(
+ tool_name: str, arguments: dict, unsupported: list[str], missing: list[str]
+) -> None:
+ payload = await _call(tool_name, arguments)
+
+ assert payload["ok"] is False
+ error = payload["error"]
+ assert error["code"] == "INVALID_INPUT"
+ details = error["details"]
+ assert details["unsupported_arguments"] == unsupported
+ assert details["missing_required_arguments"] == missing
+ # The rejection names the accepted arguments so the model can self-correct.
+ assert details["expected_arguments"]
+ for bad in unsupported:
+ assert bad not in details["expected_arguments"]
+
+
+async def test_valid_argument_shape_is_not_blocked() -> None:
+ """A validly-shaped call is not short-circuited by the middleware.
+
+ ``skyvern_get_errors`` accepts ``text``; with no browser session it fails
+ downstream, but with a tool-owned error (not the middleware's INVALID_INPUT
+ unsupported-argument rejection), proving the pre-check let it through.
+ """
+ payload = await _call("skyvern_get_errors", {"text": "boom"})
+
+ error = payload.get("error")
+ if error is not None:
+ assert "unsupported_arguments" not in error.get("details", {})
diff --git a/tests/unit/test_mcp_telemetry.py b/tests/unit/test_mcp_telemetry.py
index 797f41350..5cae890d8 100644
--- a/tests/unit/test_mcp_telemetry.py
+++ b/tests/unit/test_mcp_telemetry.py
@@ -155,6 +155,38 @@ async def test_mcp_tool_call_marks_error_results_as_not_ok() -> None:
assert "error_message" not in payload
+async def test_mcp_tool_call_reads_ok_from_structured_content() -> None:
+ """A ToolResult reporting failure via structured_content (no `data`) is recorded ok=False.
+
+ The argument-validation middleware short-circuits with exactly this shape.
+ """
+ events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []
+
+ def fake_capture(
+ event: str,
+ data: dict | None = None,
+ distinct_id: str | None = None,
+ api_key: str | None = None,
+ host: str | None = None,
+ ) -> None:
+ events.append((event, data, distinct_id, api_key, host))
+
+ context = MiddlewareContext(message=SimpleNamespace(name="skyvern_get_errors"), fastmcp_context=None)
+ response = SimpleNamespace(structured_content={"ok": False, "error": {"code": "INVALID_INPUT"}}, content=[])
+
+ async def call_next(_context: MiddlewareContext[object]) -> object:
+ return response
+
+ with patch.object(analytics, "capture", side_effect=fake_capture):
+ await MCPTelemetryMiddleware().on_call_tool(context, call_next)
+
+ assert len(events) == 1
+ _, payload, _, _, _ = events[0]
+ assert payload is not None
+ assert payload["ok"] is False
+
+
@pytest.mark.asyncio
async def test_mcp_tool_call_exception_omits_error_message() -> None:
events: list[tuple[str, dict | None, str | None, str | None, str | None]] = []