Building AI agents, atomically
fix(mcp): improve error handling and add tests for typed output schemas
KennyVaneetvelde1a7a95c0472a
diff --git a/atomic-agents/atomic_agents/connectors/mcp/mcp_factory.py b/atomic-agents/atomic_agents/connectors/mcp/mcp_factory.pyindex b401352..5bd34eb 100644--- a/atomic-agents/atomic_agents/connectors/mcp/mcp_factory.py+++ b/atomic-agents/atomic_agents/connectors/mcp/mcp_factory.py@@ -1,7 +1,7 @@import asyncioimport jsonimport logging-from typing import Any, List, Type, Optional, Union, Tuple, cast+from typing import Any, Dict, List, Type, Optional, Union, Tuple, castfrom contextlib import AsyncExitStackimport shleximport types@@ -30,7 +30,11 @@ logger = logging.getLogger(__name__)class MCPToolOutputSchema(BaseIOSchema):- """Generic output schema for dynamically generated MCP tools."""+ """Generic output schema for dynamically generated MCP tools.++ Used as a fallback when the MCP server does not provide an outputSchema definition.+ Tools with MCP-provided outputSchema will have typed output schemas instead.+ """result: Any = Field(..., description="The result returned by the MCP tool.")@@ -144,8 +148,10 @@ class MCPFactory:attribute_type=MCPAttributeType.TOOL,)- # Create output schema - use MCP-provided schema if available, otherwise fallback to generic- output_schema_dict = definition.output_schema+ # Create output schema - use MCP-provided schema if available, otherwise fallback to generic.+ # When a typed output schema is used, _has_typed_output_schema is set on the tool class+ # to enable structured content extraction at runtime (see result processing below).+ output_schema_dict: Optional[Dict[str, Any]] = definition.output_schemahas_typed_output_schema = Falseif output_schema_dict:# Use the schema transformer to create a proper typed output schema@@ -233,7 +239,13 @@ class MCPFactory:# Legacy behaviour – open a fresh connection per invocation.tool_result = await _connect_and_call()- # Process the result based on whether we have a typed output schema+ # Process the result based on whether we have a typed output schema.+ # Extraction precedence for typed schemas:+ # 1. structuredContent attribute (MCP spec primary path)+ # 2. content[0].text parsed as JSON (some servers return JSON as text)+ # 3. content[0].data dict (structured data in content item)+ # 4. Dict with structuredContent/content keys+ # 5. Direct dict usage as fallbackhas_typed_schema = getattr(self, "_has_typed_output_schema", False)BoundOutputSchema = self.output_schema@@ -245,13 +257,24 @@ class MCPFactory:structured_data = tool_result.structuredContentif isinstance(structured_data, dict):return BoundOutputSchema(**structured_data)- return BoundOutputSchema(- **structured_data.model_dump() if hasattr(structured_data, "model_dump") else {}- )+ elif hasattr(structured_data, "model_dump"):+ return BoundOutputSchema(**structured_data.model_dump())+ else:+ # Unexpected type for structuredContent+ logger.error(+ f"Unexpected structuredContent type for tool '{bound_tool_name}': "+ f"got {type(structured_data).__name__}, expected dict or BaseModel. "+ f"Content: {structured_data!r}"+ )+ raise TypeError(+ f"MCP tool '{bound_tool_name}' returned structuredContent with unexpected type "+ f"{type(structured_data).__name__}. Expected dict or BaseModel."+ )elif isinstance(tool_result, BaseModel) and hasattr(tool_result, "content"):# Try to parse content as structured datacontent = tool_result.content- if content and len(content) > 0:+ # Ensure content is a list/tuple before indexing+ if content and isinstance(content, (list, tuple)) and len(content) > 0:first_content = content[0]# Check for text content that might be JSONif hasattr(first_content, "text"):@@ -259,8 +282,23 @@ class MCPFactory:parsed = json.loads(first_content.text)if isinstance(parsed, dict):return BoundOutputSchema(**parsed)- except (json.JSONDecodeError, TypeError):- pass+ else:+ logger.debug(+ f"Tool '{bound_tool_name}' content parsed as JSON but was "+ f"{type(parsed).__name__}, not dict. Trying other extraction methods."+ )+ except json.JSONDecodeError as e:+ logger.debug(+ f"Tool '{bound_tool_name}' content is not valid JSON: {e}. "+ f"Content preview: {first_content.text[:200]!r}..."+ if len(first_content.text) > 200+ else f"Content: {first_content.text!r}"+ )+ except TypeError as e:+ logger.warning(+ f"Tool '{bound_tool_name}' content.text has unexpected type "+ f"{type(first_content.text).__name__}: {e}"+ )# Check for structured content in the content itemif hasattr(first_content, "data") and isinstance(first_content.data, dict):return BoundOutputSchema(**first_content.data)@@ -274,12 +312,21 @@ class MCPFactory:# Fallback: try to use tool_result directly if it's a dictif isinstance(tool_result, dict):return BoundOutputSchema(**tool_result)- # Last resort: return with whatever we can extract- logger.warning(- f"Could not parse structured output for tool '{bound_tool_name}', falling back to generic handling"+ # If we have a typed schema but couldn't extract structured content,+ # this is an error - we cannot fall through to generic handling+ # because the typed schema doesn't have a 'result' field.+ logger.error(+ f"Could not parse structured output for tool '{bound_tool_name}'. "+ f"Expected typed output but got: type={type(tool_result).__name__}, "+ f"value={tool_result!r}"+ )+ raise ValueError(+ f"MCP tool '{bound_tool_name}' has outputSchema but returned unparseable result. "+ f"Received type: {type(tool_result).__name__}. "+ f"Check MCP server implementation.")- # Generic output schema handling (original behavior)+ # Generic output schema handling (original behavior) - only for tools without typed schemasif isinstance(tool_result, BaseModel) and hasattr(tool_result, "content"):actual_result_content = tool_result.contentelif isinstance(tool_result, dict) and "content" in tool_result:@@ -791,7 +838,6 @@ class MCPFactory:texts = []for message in messages:- # content = getattr(m, 'content', None)if isinstance(message, BaseModel) and hasattr(message, "content"):content = message.content # type: ignoreelif isinstance(message, dict) and "content" in message:diff --git a/atomic-agents/atomic_agents/connectors/mcp/schema_transformer.py b/atomic-agents/atomic_agents/connectors/mcp/schema_transformer.pyindex ee8e1e5..f2527ec 100644--- a/atomic-agents/atomic_agents/connectors/mcp/schema_transformer.py+++ b/atomic-agents/atomic_agents/connectors/mcp/schema_transformer.py@@ -162,7 +162,10 @@ class SchemaTransformer:tool_name_literal: Tool name to use for the Literal typedocstring: Optional docstring for the modelattribute_type: Type of MCP attribute (tool, resource, prompt)- is_output_schema: If True, skip adding the tool_name literal field (output schemas don't need it)+ is_output_schema: If True, skip adding the tool_name/resource_name/prompt_name literal field.+ Output schemas represent tool responses and don't need an identifier field since+ the tool has already been selected and executed. Input schemas need the identifier+ for discriminated unions when selecting among multiple tools in an orchestrator.Returns:Pydantic model classdiff --git a/atomic-agents/tests/connectors/mcp/test_mcp_factory.py b/atomic-agents/tests/connectors/mcp/test_mcp_factory.pyindex 526fa8d..3ac5c19 100644--- a/atomic-agents/tests/connectors/mcp/test_mcp_factory.py+++ b/atomic-agents/tests/connectors/mcp/test_mcp_factory.py@@ -165,6 +165,383 @@ def test_fetch_mcp_tools_mixed_output_schemas(monkeypatch):assert "result" not in typed_tool.output_schema.model_fields+# =============================================================================+# Tests for typed output schema result processing+# =============================================================================+++class MockStructuredContentResult(BaseModel):+ """Mock MCP result with structuredContent attribute (MCP spec primary path)"""++ structuredContent: dict+++class MockContentItem(BaseModel):+ """Mock content item with text attribute"""++ text: str+++class MockContentItemWithData(BaseModel):+ """Mock content item with data attribute"""++ data: dict+++class MockContentResult(BaseModel):+ """Mock MCP result with content array"""++ content: list+++async def test_typed_output_schema_with_structured_content_dict(monkeypatch):+ """Test result processing when tool returns BaseModel with structuredContent as dict"""+ input_schema = {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}+ output_schema = {+ "type": "object",+ "properties": {+ "results": {"type": "array", "items": {"type": "string"}},+ "count": {"type": "integer"},+ },+ "required": ["results", "count"],+ }+ definitions = [+ MCPToolDefinition(+ name="SearchTool", description="Search tool", input_schema=input_schema, output_schema=output_schema+ )+ ]+ monkeypatch.setattr(MCPFactory, "_fetch_tool_definitions", lambda self: definitions)+ tools = fetch_mcp_tools("http://example.com", MCPTransportType.HTTP_STREAM)+ tool_cls = tools[0]++ tool_instance = tool_cls()+ mock_result = MockStructuredContentResult(structuredContent={"results": ["a", "b"], "count": 2})++ import atomic_agents.connectors.mcp.mcp_factory as factory_module++ class MockClientSession:+ def __init__(self, *args):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *args):+ pass++ async def initialize(self):+ pass++ async def call_tool(self, name, arguments):+ return mock_result++ class MockHttpClient:+ async def __aenter__(self):+ return (None, None, None)++ async def __aexit__(self, *args):+ pass++ monkeypatch.setattr(factory_module, "ClientSession", MockClientSession)+ monkeypatch.setattr(factory_module, "streamablehttp_client", lambda *args, **kwargs: MockHttpClient())++ InputSchema = tool_cls.input_schema+ params = InputSchema(tool_name="SearchTool", query="test")++ result = await tool_instance.arun(params)++ assert result.results == ["a", "b"]+ assert result.count == 2+++async def test_typed_output_schema_with_json_text_content(monkeypatch):+ """Test result processing when tool returns content[0].text as JSON string"""+ input_schema = {"type": "object", "properties": {}, "required": []}+ output_schema = {+ "type": "object",+ "properties": {"data": {"type": "string"}},+ "required": ["data"],+ }+ definitions = [+ MCPToolDefinition(name="JsonTool", description="JSON tool", input_schema=input_schema, output_schema=output_schema)+ ]+ monkeypatch.setattr(MCPFactory, "_fetch_tool_definitions", lambda self: definitions)+ tools = fetch_mcp_tools("http://example.com", MCPTransportType.HTTP_STREAM)+ tool_cls = tools[0]++ tool_instance = tool_cls()+ mock_content_item = MockContentItem(text='{"data": "hello"}')+ mock_result = MockContentResult(content=[mock_content_item])++ import atomic_agents.connectors.mcp.mcp_factory as factory_module++ class MockClientSession:+ def __init__(self, *args):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *args):+ pass++ async def initialize(self):+ pass++ async def call_tool(self, name, arguments):+ return mock_result++ class MockHttpClient:+ async def __aenter__(self):+ return (None, None, None)++ async def __aexit__(self, *args):+ pass++ monkeypatch.setattr(factory_module, "ClientSession", MockClientSession)+ monkeypatch.setattr(factory_module, "streamablehttp_client", lambda *args, **kwargs: MockHttpClient())++ InputSchema = tool_cls.input_schema+ params = InputSchema(tool_name="JsonTool")++ result = await tool_instance.arun(params)++ assert result.data == "hello"+++async def test_typed_output_schema_with_content_data_dict(monkeypatch):+ """Test result processing when content item has .data attribute as dict"""+ input_schema = {"type": "object", "properties": {}, "required": []}+ output_schema = {+ "type": "object",+ "properties": {"value": {"type": "integer"}},+ "required": ["value"],+ }+ definitions = [+ MCPToolDefinition(name="DataTool", description="Data tool", input_schema=input_schema, output_schema=output_schema)+ ]+ monkeypatch.setattr(MCPFactory, "_fetch_tool_definitions", lambda self: definitions)+ tools = fetch_mcp_tools("http://example.com", MCPTransportType.HTTP_STREAM)+ tool_cls = tools[0]++ tool_instance = tool_cls()+ mock_content_item = MockContentItemWithData(data={"value": 42})+ mock_result = MockContentResult(content=[mock_content_item])++ import atomic_agents.connectors.mcp.mcp_factory as factory_module++ class MockClientSession:+ def __init__(self, *args):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *args):+ pass++ async def initialize(self):+ pass++ async def call_tool(self, name, arguments):+ return mock_result++ class MockHttpClient:+ async def __aenter__(self):+ return (None, None, None)++ async def __aexit__(self, *args):+ pass++ monkeypatch.setattr(factory_module, "ClientSession", MockClientSession)+ monkeypatch.setattr(factory_module, "streamablehttp_client", lambda *args, **kwargs: MockHttpClient())++ InputSchema = tool_cls.input_schema+ params = InputSchema(tool_name="DataTool")++ result = await tool_instance.arun(params)++ assert result.value == 42+++async def test_typed_output_schema_with_raw_dict(monkeypatch):+ """Test fallback when tool_result is plain dict"""+ input_schema = {"type": "object", "properties": {}, "required": []}+ output_schema = {+ "type": "object",+ "properties": {"name": {"type": "string"}},+ "required": ["name"],+ }+ definitions = [+ MCPToolDefinition(name="DictTool", description="Dict tool", input_schema=input_schema, output_schema=output_schema)+ ]+ monkeypatch.setattr(MCPFactory, "_fetch_tool_definitions", lambda self: definitions)+ tools = fetch_mcp_tools("http://example.com", MCPTransportType.HTTP_STREAM)+ tool_cls = tools[0]++ tool_instance = tool_cls()+ mock_result = {"name": "test_value"}++ import atomic_agents.connectors.mcp.mcp_factory as factory_module++ class MockClientSession:+ def __init__(self, *args):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *args):+ pass++ async def initialize(self):+ pass++ async def call_tool(self, name, arguments):+ return mock_result++ class MockHttpClient:+ async def __aenter__(self):+ return (None, None, None)++ async def __aexit__(self, *args):+ pass++ monkeypatch.setattr(factory_module, "ClientSession", MockClientSession)+ monkeypatch.setattr(factory_module, "streamablehttp_client", lambda *args, **kwargs: MockHttpClient())++ InputSchema = tool_cls.input_schema+ params = InputSchema(tool_name="DictTool")++ result = await tool_instance.arun(params)++ assert result.name == "test_value"+++async def test_typed_output_schema_raises_on_unparseable_result(monkeypatch):+ """Test that ValueError is raised when typed schema tool returns unparseable result"""+ input_schema = {"type": "object", "properties": {}, "required": []}+ output_schema = {+ "type": "object",+ "properties": {"data": {"type": "string"}},+ "required": ["data"],+ }+ definitions = [+ MCPToolDefinition(+ name="FailingTool", description="Failing tool", input_schema=input_schema, output_schema=output_schema+ )+ ]+ monkeypatch.setattr(MCPFactory, "_fetch_tool_definitions", lambda self: definitions)+ tools = fetch_mcp_tools("http://example.com", MCPTransportType.HTTP_STREAM)+ tool_cls = tools[0]++ tool_instance = tool_cls()+ # Return a string which can't be parsed as structured content+ mock_result = "just a string, not structured"++ import atomic_agents.connectors.mcp.mcp_factory as factory_module++ class MockClientSession:+ def __init__(self, *args):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *args):+ pass++ async def initialize(self):+ pass++ async def call_tool(self, name, arguments):+ return mock_result++ class MockHttpClient:+ async def __aenter__(self):+ return (None, None, None)++ async def __aexit__(self, *args):+ pass++ monkeypatch.setattr(factory_module, "ClientSession", MockClientSession)+ monkeypatch.setattr(factory_module, "streamablehttp_client", lambda *args, **kwargs: MockHttpClient())++ InputSchema = tool_cls.input_schema+ params = InputSchema(tool_name="FailingTool")++ with pytest.raises(RuntimeError) as exc_info:+ await tool_instance.arun(params)++ # The ValueError gets wrapped in RuntimeError by the outer exception handler+ assert "unparseable result" in str(exc_info.value) or "FailingTool" in str(exc_info.value)+++async def test_typed_output_schema_handles_empty_content_array(monkeypatch):+ """Test graceful handling when content array is empty"""+ input_schema = {"type": "object", "properties": {}, "required": []}+ output_schema = {+ "type": "object",+ "properties": {"data": {"type": "string"}},+ "required": ["data"],+ }+ definitions = [+ MCPToolDefinition(+ name="EmptyContentTool", description="Empty content tool", input_schema=input_schema, output_schema=output_schema+ )+ ]+ monkeypatch.setattr(MCPFactory, "_fetch_tool_definitions", lambda self: definitions)+ tools = fetch_mcp_tools("http://example.com", MCPTransportType.HTTP_STREAM)+ tool_cls = tools[0]++ tool_instance = tool_cls()+ # Empty content array - should fall through and raise error+ mock_result = MockContentResult(content=[])++ import atomic_agents.connectors.mcp.mcp_factory as factory_module++ class MockClientSession:+ def __init__(self, *args):+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, *args):+ pass++ async def initialize(self):+ pass++ async def call_tool(self, name, arguments):+ return mock_result++ class MockHttpClient:+ async def __aenter__(self):+ return (None, None, None)++ async def __aexit__(self, *args):+ pass++ monkeypatch.setattr(factory_module, "ClientSession", MockClientSession)+ monkeypatch.setattr(factory_module, "streamablehttp_client", lambda *args, **kwargs: MockHttpClient())++ InputSchema = tool_cls.input_schema+ params = InputSchema(tool_name="EmptyContentTool")++ # Should raise error since we can't extract structured content+ with pytest.raises(RuntimeError) as exc_info:+ await tool_instance.arun(params)++ assert "EmptyContentTool" in str(exc_info.value)++def test_fetch_mcp_resources_with_definitions_stdio(monkeypatch):input_schema = {"type": "object", "properties": {}, "required": []}uri = "resource://example-resource"diff --git a/atomic-agents/tests/connectors/mcp/test_schema_transformer.py b/atomic-agents/tests/connectors/mcp/test_schema_transformer.pyindex 820ed0e..c198c9b 100644--- a/atomic-agents/tests/connectors/mcp/test_schema_transformer.py+++ b/atomic-agents/tests/connectors/mcp/test_schema_transformer.py@@ -207,3 +207,74 @@ class TestCreateModelFromSchema:assert hasattr(objects_field.annotation, "__origin__") and objects_field.annotation.__origin__ is listinner_type = objects_field.annotation.__args__[0]assert inner_type != Any++ def test_output_schema_no_tool_name_field(self):+ """Test that output schemas don't include tool_name field when is_output_schema=True."""+ schema = {+ "type": "object",+ "properties": {+ "results": {"type": "array", "items": {"type": "string"}, "description": "Search results"},+ "count": {"type": "integer", "description": "Number of results"},+ },+ "required": ["results", "count"],+ }+ model = SchemaTransformer.create_model_from_schema(+ schema, "OutputModel", "my_tool", is_output_schema=True+ )++ # Output schema should NOT have tool_name field+ assert "tool_name" not in model.model_fields+ # But should have the defined fields+ assert "results" in model.model_fields+ assert "count" in model.model_fields+ assert len(model.model_fields) == 2 # Only results and count, no tool_name++ # Should be instantiable without tool_name+ instance = model(results=["a", "b"], count=2)+ assert instance.results == ["a", "b"]+ assert instance.count == 2++ def test_input_schema_has_tool_name_field(self):+ """Test that input schemas include tool_name field when is_output_schema=False (default)."""+ schema = {+ "type": "object",+ "properties": {+ "query": {"type": "string", "description": "Search query"},+ },+ "required": ["query"],+ }+ model = SchemaTransformer.create_model_from_schema(+ schema, "InputModel", "my_tool", is_output_schema=False+ )++ # Input schema SHOULD have tool_name field+ assert "tool_name" in model.model_fields+ assert "query" in model.model_fields+ assert len(model.model_fields) == 2 # query and tool_name++ # Should require tool_name for instantiation+ instance = model(tool_name="my_tool", query="test")+ assert instance.tool_name == "my_tool"+ assert instance.query == "test"++ def test_output_schema_with_resource_attribute_type(self):+ """Test that output schemas work with different attribute types."""+ from atomic_agents.connectors.mcp.mcp_definition_service import MCPAttributeType++ schema = {+ "type": "object",+ "properties": {+ "data": {"type": "string", "description": "Some data"},+ },+ "required": ["data"],+ }++ # Output schema for resource - should not have resource_name+ model = SchemaTransformer.create_model_from_schema(+ schema, "ResourceOutput", "my_resource",+ attribute_type=MCPAttributeType.RESOURCE,+ is_output_schema=True+ )++ assert "resource_name" not in model.model_fields+ assert "data" in model.model_fields