The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
standard-tests: add test for structured output (#23631)
ccurme390ee8d9715b
diff --git a/libs/partners/groq/tests/integration_tests/test_standard.py b/libs/partners/groq/tests/integration_tests/test_standard.pyindex 4234ff616c..290fef253a 100644--- a/libs/partners/groq/tests/integration_tests/test_standard.py+++ b/libs/partners/groq/tests/integration_tests/test_standard.py@@ -9,7 +9,7 @@ from langchain_standard_tests.integration_tests import ChatModelIntegrationTestsfrom langchain_groq import ChatGroq-class TestGroqStandard(ChatModelIntegrationTests):+class BaseTestGroq(ChatModelIntegrationTests):@propertydef chat_model_class(self) -> Type[BaseChatModel]:return ChatGroq@@ -17,3 +17,32 @@ class TestGroqStandard(ChatModelIntegrationTests):@pytest.mark.xfail(reason="Not yet implemented.")def test_tool_message_histories_list_content(self, model: BaseChatModel) -> None:super().test_tool_message_histories_list_content(model)+++class TestGroqMixtral(BaseTestGroq):+ @property+ def chat_model_params(self) -> dict:+ return {+ "temperature": 0,+ }++ @pytest.mark.xfail(+ reason=("Fails with 'Failed to call a function. Please adjust your prompt.'")+ )+ def test_structured_output(self, model: BaseChatModel) -> None:+ super().test_structured_output(model)+++class TestGroqLlama(BaseTestGroq):+ @property+ def chat_model_params(self) -> dict:+ return {+ "model": "llama3-8b-8192",+ "temperature": 0,+ }++ @pytest.mark.xfail(+ reason=("Fails with 'Failed to call a function. Please adjust your prompt.'")+ )+ def test_tool_message_histories_string_content(self, model: BaseChatModel) -> None:+ super().test_tool_message_histories_string_content(model)diff --git a/libs/partners/openai/langchain_openai/chat_models/azure.py b/libs/partners/openai/langchain_openai/chat_models/azure.pyindex fe56620383..a13ef2b385 100644--- a/libs/partners/openai/langchain_openai/chat_models/azure.py+++ b/libs/partners/openai/langchain_openai/chat_models/azure.py@@ -3,15 +3,35 @@ from __future__ import annotationsimport loggingimport os-from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Type, Union+from operator import itemgetter+from typing import (+ Any,+ Callable,+ Dict,+ List,+ Literal,+ Optional,+ Sequence,+ Type,+ TypedDict,+ TypeVar,+ Union,+ overload,+)import openaifrom langchain_core.language_models import LanguageModelInputfrom langchain_core.language_models.chat_models import LangSmithParamsfrom langchain_core.messages import BaseMessage+from langchain_core.output_parsers import JsonOutputParser, PydanticOutputParser+from langchain_core.output_parsers.base import OutputParserLike+from langchain_core.output_parsers.openai_tools import (+ JsonOutputKeyToolsParser,+ PydanticToolsParser,+)from langchain_core.outputs import ChatResultfrom langchain_core.pydantic_v1 import BaseModel, Field, SecretStr, root_validator-from langchain_core.runnables import Runnable+from langchain_core.runnables import Runnable, RunnableMap, RunnablePassthroughfrom langchain_core.tools import BaseToolfrom langchain_core.utils import convert_to_secret_str, get_from_dict_or_envfrom langchain_core.utils.function_calling import convert_to_openai_tool@@ -21,6 +41,21 @@ from langchain_openai.chat_models.base import BaseChatOpenAIlogger = logging.getLogger(__name__)+_BM = TypeVar("_BM", bound=BaseModel)+_DictOrPydanticClass = Union[Dict[str, Any], Type[_BM]]+_DictOrPydantic = Union[Dict, _BM]+++class _AllReturnType(TypedDict):+ raw: BaseMessage+ parsed: Optional[_DictOrPydantic]+ parsing_error: Optional[BaseException]+++def _is_pydantic_class(obj: Any) -> bool:+ return isinstance(obj, type) and issubclass(obj, BaseModel)++class AzureChatOpenAI(BaseChatOpenAI):"""`Azure OpenAI` Chat Completion API.@@ -233,6 +268,250 @@ class AzureChatOpenAI(BaseChatOpenAI):tool_choice = convert_to_openai_tool(tools[0])["function"]["name"]return super().bind_tools(tools, tool_choice=tool_choice, **kwargs)+ # TODO: Fix typing.+ @overload # type: ignore[override]+ def with_structured_output(+ self,+ schema: Optional[_DictOrPydanticClass] = None,+ *,+ method: Literal["function_calling", "json_mode"] = "function_calling",+ include_raw: Literal[True] = True,+ **kwargs: Any,+ ) -> Runnable[LanguageModelInput, _AllReturnType]:+ ...++ @overload+ def with_structured_output(+ self,+ schema: Optional[_DictOrPydanticClass] = None,+ *,+ method: Literal["function_calling", "json_mode"] = "function_calling",+ include_raw: Literal[False] = False,+ **kwargs: Any,+ ) -> Runnable[LanguageModelInput, _DictOrPydantic]:+ ...++ def with_structured_output(+ self,+ schema: Optional[_DictOrPydanticClass] = None,+ *,+ method: Literal["function_calling", "json_mode"] = "function_calling",+ include_raw: bool = False,+ **kwargs: Any,+ ) -> Runnable[LanguageModelInput, _DictOrPydantic]:+ """Model wrapper that returns outputs formatted to match the given schema.++ Args:+ schema: The output schema as a dict or a Pydantic class. If a Pydantic class+ then the model output will be an object of that class. If a dict then+ the model output will be a dict. With a Pydantic class the returned+ attributes will be validated, whereas with a dict they will not be. If+ `method` is "function_calling" and `schema` is a dict, then the dict+ must match the OpenAI function-calling spec or be a valid JSON schema+ with top level 'title' and 'description' keys specified.+ method: The method for steering model generation, either "function_calling"+ or "json_mode". If "function_calling" then the schema will be converted+ to an OpenAI function and the returned model will make use of the+ function-calling API. If "json_mode" then OpenAI's JSON mode will be+ used. Note that if using "json_mode" then you must include instructions+ for formatting the output into the desired schema into the model call.+ include_raw: If False then only the parsed structured output is returned. If+ an error occurs during model output parsing it will be raised. If True+ then both the raw model response (a BaseMessage) and the parsed model+ response will be returned. If an error occurs during output parsing it+ will be caught and returned as well. The final output is always a dict+ with keys "raw", "parsed", and "parsing_error".++ Returns:+ A Runnable that takes any ChatModel input and returns as output:++ If include_raw is True then a dict with keys:+ raw: BaseMessage+ parsed: Optional[_DictOrPydantic]+ parsing_error: Optional[BaseException]++ If include_raw is False then just _DictOrPydantic is returned,+ where _DictOrPydantic depends on the schema:++ If schema is a Pydantic class then _DictOrPydantic is the Pydantic+ class.++ If schema is a dict then _DictOrPydantic is a dict.++ Example: Function-calling, Pydantic schema (method="function_calling", include_raw=False):+ .. code-block:: python++ from langchain_openai import AzureChatOpenAI+ from langchain_core.pydantic_v1 import BaseModel+++ class AnswerWithJustification(BaseModel):+ '''An answer to the user question along with justification for the answer.'''++ answer: str+ justification: str+++ llm = AzureChatOpenAI(azure_deployment="gpt-35-turbo", temperature=0)+ structured_llm = llm.with_structured_output(AnswerWithJustification)++ structured_llm.invoke(+ "What weighs more a pound of bricks or a pound of feathers"+ )++ # -> AnswerWithJustification(+ # answer='They weigh the same',+ # justification='Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ.'+ # )++ Example: Function-calling, Pydantic schema (method="function_calling", include_raw=True):+ .. code-block:: python++ from langchain_openai import AzureChatOpenAI+ from langchain_core.pydantic_v1 import BaseModel+++ class AnswerWithJustification(BaseModel):+ '''An answer to the user question along with justification for the answer.'''++ answer: str+ justification: str+++ llm = AzureChatOpenAI(azure_deployment="gpt-35-turbo", temperature=0)+ structured_llm = llm.with_structured_output(+ AnswerWithJustification, include_raw=True+ )++ structured_llm.invoke(+ "What weighs more a pound of bricks or a pound of feathers"+ )+ # -> {+ # 'raw': AIMessage(content='', additional_kwargs={'tool_calls': [{'id': 'call_Ao02pnFYXD6GN1yzc0uXPsvF', 'function': {'arguments': '{"answer":"They weigh the same.","justification":"Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ."}', 'name': 'AnswerWithJustification'}, 'type': 'function'}]}),+ # 'parsed': AnswerWithJustification(answer='They weigh the same.', justification='Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ.'),+ # 'parsing_error': None+ # }++ Example: Function-calling, dict schema (method="function_calling", include_raw=False):+ .. code-block:: python++ from langchain_openai import AzureChatOpenAI+ from langchain_core.pydantic_v1 import BaseModel+ from langchain_core.utils.function_calling import convert_to_openai_tool+++ class AnswerWithJustification(BaseModel):+ '''An answer to the user question along with justification for the answer.'''++ answer: str+ justification: str+++ dict_schema = convert_to_openai_tool(AnswerWithJustification)+ llm = AzureChatOpenAI(azure_deployment="gpt-35-turbo", temperature=0)+ structured_llm = llm.with_structured_output(dict_schema)++ structured_llm.invoke(+ "What weighs more a pound of bricks or a pound of feathers"+ )+ # -> {+ # 'answer': 'They weigh the same',+ # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume and density of the two substances differ.'+ # }++ Example: JSON mode, Pydantic schema (method="json_mode", include_raw=True):+ .. code-block::++ from langchain_openai import AzureChatOpenAI+ from langchain_core.pydantic_v1 import BaseModel++ class AnswerWithJustification(BaseModel):+ answer: str+ justification: str++ llm = AzureChatOpenAI(azure_deployment="gpt-35-turbo", temperature=0)+ structured_llm = llm.with_structured_output(+ AnswerWithJustification,+ method="json_mode",+ include_raw=True+ )++ structured_llm.invoke(+ "Answer the following question. "+ "Make sure to return a JSON blob with keys 'answer' and 'justification'.\n\n"+ "What's heavier a pound of bricks or a pound of feathers?"+ )+ # -> {+ # 'raw': AIMessage(content='{\n "answer": "They are both the same weight.",\n "justification": "Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight." \n}'),+ # 'parsed': AnswerWithJustification(answer='They are both the same weight.', justification='Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight.'),+ # 'parsing_error': None+ # }++ Example: JSON mode, no schema (schema=None, method="json_mode", include_raw=True):+ .. code-block::++ structured_llm = llm.with_structured_output(method="json_mode", include_raw=True)++ structured_llm.invoke(+ "Answer the following question. "+ "Make sure to return a JSON blob with keys 'answer' and 'justification'.\n\n"+ "What's heavier a pound of bricks or a pound of feathers?"+ )+ # -> {+ # 'raw': AIMessage(content='{\n "answer": "They are both the same weight.",\n "justification": "Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight." \n}'),+ # 'parsed': {+ # 'answer': 'They are both the same weight.',+ # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight.'+ # },+ # 'parsing_error': None+ # }+++ """ # noqa: E501+ if kwargs:+ raise ValueError(f"Received unsupported arguments {kwargs}")+ is_pydantic_schema = _is_pydantic_class(schema)+ if method == "function_calling":+ if schema is None:+ raise ValueError(+ "schema must be specified when method is 'function_calling'. "+ "Received None."+ )+ llm = self.bind_tools([schema], tool_choice=True)+ if is_pydantic_schema:+ output_parser: OutputParserLike = PydanticToolsParser(+ tools=[schema], first_tool_only=True+ )+ else:+ key_name = convert_to_openai_tool(schema)["function"]["name"]+ output_parser = JsonOutputKeyToolsParser(+ key_name=key_name, first_tool_only=True+ )+ elif method == "json_mode":+ llm = self.bind(response_format={"type": "json_object"})+ output_parser = (+ PydanticOutputParser(pydantic_object=schema)+ if is_pydantic_schema+ else JsonOutputParser()+ )+ else:+ raise ValueError(+ f"Unrecognized method argument. Expected one of 'function_calling' or "+ f"'json_mode'. Received: '{method}'"+ )++ if include_raw:+ parser_assign = RunnablePassthrough.assign(+ parsed=itemgetter("raw") | output_parser, parsing_error=lambda _: None+ )+ parser_none = RunnablePassthrough.assign(parsed=lambda _: None)+ parser_with_fallback = parser_assign.with_fallbacks(+ [parser_none], exception_key="parsing_error"+ )+ return RunnableMap(raw=llm) | parser_with_fallback+ else:+ return llm | output_parser+@propertydef _identifying_params(self) -> Dict[str, Any]:"""Get the identifying parameters."""diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.pyindex b9c758024a..1a1f9561ed 100644--- a/libs/partners/openai/langchain_openai/chat_models/base.py+++ b/libs/partners/openai/langchain_openai/chat_models/base.py@@ -1113,8 +1113,6 @@ class BaseChatOpenAI(BaseChatModel):Example: JSON mode, no schema (schema=None, method="json_mode", include_raw=True):.. code-block::- from langchain_openai import ChatOpenAI-structured_llm = llm.with_structured_output(method="json_mode", include_raw=True)structured_llm.invoke(diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_azure.py b/libs/partners/openai/tests/integration_tests/chat_models/test_azure.pyindex 63f2eac450..c44e7a3b2f 100644--- a/libs/partners/openai/tests/integration_tests/chat_models/test_azure.py+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_azure.py@@ -12,7 +12,6 @@ from langchain_core.messages import (HumanMessage,)from langchain_core.outputs import ChatGeneration, ChatResult, LLMResult-from langchain_core.pydantic_v1 import BaseModelfrom langchain_openai import AzureChatOpenAIfrom tests.unit_tests.fake.callbacks import FakeCallbackHandler@@ -226,18 +225,3 @@ def test_openai_invoke(llm: AzureChatOpenAI) -> None:result = llm.invoke("I'm Pickle Rick", config=dict(tags=["foo"]))assert isinstance(result.content, str)assert result.response_metadata.get("model_name") is not None--[email protected](reason="Need tool calling model deployed on azure")-def test_openai_structured_output(llm: AzureChatOpenAI) -> None:- class MyModel(BaseModel):- """A Person"""-- name: str- age: int-- llm_structure = llm.with_structured_output(MyModel)- result = llm_structure.invoke("I'm a 27 year old named Erick")- assert isinstance(result, MyModel)- assert result.name == "Erick"- assert result.age == 27diff --git a/libs/standard-tests/langchain_standard_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_standard_tests/integration_tests/chat_models.pyindex 90ef83fd4f..ae98167a7e 100644--- a/libs/standard-tests/langchain_standard_tests/integration_tests/chat_models.py+++ b/libs/standard-tests/langchain_standard_tests/integration_tests/chat_models.py@@ -12,6 +12,7 @@ from langchain_core.messages import (HumanMessage,ToolMessage,)+from langchain_core.pydantic_v1 import BaseModel, Fieldfrom langchain_core.tools import toolfrom langchain_standard_tests.unit_tests.chat_models import (@@ -139,6 +140,20 @@ class ChatModelIntegrationTests(ChatModelTests):assert isinstance(full, AIMessage)_validate_tool_call_message(full)+ def test_structured_output(self, model: BaseChatModel) -> None:+ if not self.has_tool_calling:+ pytest.skip("Test requires tool calling.")++ class Joke(BaseModel):+ """Joke to tell user."""++ setup: str = Field(description="question to set up a joke")+ punchline: str = Field(description="answer to resolve the joke")++ chat = model.with_structured_output(Joke)+ result = chat.invoke("Tell me a joke about cats.")+ assert isinstance(result, Joke)+def test_tool_message_histories_string_content(self,model: BaseChatModel,