Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
Brandon/improve llm structured output (#2029)
Brandon Hancock (bhancock_ai)f4bb040ad8d9
diff --git a/docs/concepts/llms.mdx b/docs/concepts/llms.mdxindex 309ae8923..b6a3ac328 100644--- a/docs/concepts/llms.mdx+++ b/docs/concepts/llms.mdx@@ -720,6 +720,30 @@ Learn how to get the most out of your LLM configuration:</Accordion></AccordionGroup>+## Structured LLM Calls++CrewAI supports structured responses from LLM calls by allowing you to define a `response_format` using a Pydantic model. This enables the framework to automatically parse and validate the output, making it easier to integrate the response into your application without manual post-processing.++For example, you can define a Pydantic model to represent the expected response structure and pass it as the `response_format` when instantiating the LLM. The model will then be used to convert the LLM output into a structured Python object.++```python Code+from crewai import LLM++class Dog(BaseModel):+ name: str+ age: int+ breed: str+++llm = LLM(model="gpt-4o", response_format=Dog)++response = llm.call(+ "Analyze the following messages and return the name, age, and breed. "+ "Meet Kona! She is 3 years old and is a black german shepherd."+)+print(response)+```+## Common Issues and Solutions<Tabs>diff --git a/src/crewai/llm.py b/src/crewai/llm.pyindex 1e52773a1..d8f2be230 100644--- a/src/crewai/llm.py+++ b/src/crewai/llm.py@@ -5,15 +5,17 @@ import sysimport threadingimport warningsfrom contextlib import contextmanager-from typing import Any, Dict, List, Literal, Optional, Union, cast+from typing import Any, Dict, List, Literal, Optional, Type, Union, castfrom dotenv import load_dotenv+from pydantic import BaseModelwith warnings.catch_warnings():warnings.simplefilter("ignore", UserWarning)import litellmfrom litellm import Choices, get_supported_openai_paramsfrom litellm.types.utils import ModelResponse+ from litellm.utils import supports_response_schemafrom crewai.utilities.exceptions.context_window_exceeding_exception import (@@ -128,7 +130,7 @@ class LLM:presence_penalty: Optional[float] = None,frequency_penalty: Optional[float] = None,logit_bias: Optional[Dict[int, float]] = None,- response_format: Optional[Dict[str, Any]] = None,+ response_format: Optional[Type[BaseModel]] = None,seed: Optional[int] = None,logprobs: Optional[int] = None,top_logprobs: Optional[int] = None,@@ -213,6 +215,9 @@ class LLM:response = llm.call(messages)print(response)"""+ # Validate parameters before proceeding with the call.+ self._validate_call_params()+if isinstance(messages, str):messages = [{"role": "user", "content": messages}]@@ -312,6 +317,36 @@ class LLM:logging.error(f"LiteLLM call failed: {str(e)}")raise+ def _get_custom_llm_provider(self) -> str:+ """+ Derives the custom_llm_provider from the model string.+ - For example, if the model is "openrouter/deepseek/deepseek-chat", returns "openrouter".+ - If the model is "gemini/gemini-1.5-pro", returns "gemini".+ - If there is no '/', defaults to "openai".+ """+ if "/" in self.model:+ return self.model.split("/")[0]+ return "openai"++ def _validate_call_params(self) -> None:+ """+ Validate parameters before making a call. Currently this only checks if+ a response_format is provided and whether the model supports it.+ The custom_llm_provider is dynamically determined from the model:+ - E.g., "openrouter/deepseek/deepseek-chat" yields "openrouter"+ - "gemini/gemini-1.5-pro" yields "gemini"+ - If no slash is present, "openai" is assumed.+ """+ provider = self._get_custom_llm_provider()+ if self.response_format is not None and not supports_response_schema(+ model=self.model,+ custom_llm_provider=provider,+ ):+ raise ValueError(+ f"The model {self.model} does not support response_format for provider '{provider}'. "+ "Please remove response_format or use a supported model."+ )+def supports_function_calling(self) -> bool:try:params = get_supported_openai_params(model=self.model)diff --git a/tests/llm_test.py b/tests/llm_test.pyindex 723e28a16..d64639dca 100644--- a/tests/llm_test.py+++ b/tests/llm_test.py@@ -3,6 +3,7 @@ from time import sleepfrom unittest.mock import MagicMock, patchimport pytest+from pydantic import BaseModelfrom crewai.agents.agent_builder.utilities.base_token_process import TokenProcessfrom crewai.llm import LLM@@ -205,6 +206,52 @@ def test_llm_passes_additional_params():assert result == "Test response"+def test_get_custom_llm_provider_openrouter():+ llm = LLM(model="openrouter/deepseek/deepseek-chat")+ assert llm._get_custom_llm_provider() == "openrouter"+++def test_get_custom_llm_provider_gemini():+ llm = LLM(model="gemini/gemini-1.5-pro")+ assert llm._get_custom_llm_provider() == "gemini"+++def test_get_custom_llm_provider_openai():+ llm = LLM(model="gpt-4")+ assert llm._get_custom_llm_provider() == "openai"+++def test_validate_call_params_supported():+ class DummyResponse(BaseModel):+ a: int++ # Patch supports_response_schema to simulate a supported model.+ with patch("crewai.llm.supports_response_schema", return_value=True):+ llm = LLM(+ model="openrouter/deepseek/deepseek-chat", response_format=DummyResponse+ )+ # Should not raise any error.+ llm._validate_call_params()+++def test_validate_call_params_not_supported():+ class DummyResponse(BaseModel):+ a: int++ # Patch supports_response_schema to simulate an unsupported model.+ with patch("crewai.llm.supports_response_schema", return_value=False):+ llm = LLM(model="gemini/gemini-1.5-pro", response_format=DummyResponse)+ with pytest.raises(ValueError) as excinfo:+ llm._validate_call_params()+ assert "does not support response_format" in str(excinfo.value)+++def test_validate_call_params_no_response_format():+ # When no response_format is provided, no validation error should occur.+ llm = LLM(model="gemini/gemini-1.5-pro", response_format=None)+ llm._validate_call_params()++@pytest.mark.vcr(filter_headers=["authorization"])def test_o3_mini_reasoning_effort_high():llm = LLM(