The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
core, openai, standard-tests: improve OpenAI compatibility with Anthropic content blocks (#30128)
ccurme52b0570bec07
diff --git a/libs/core/langchain_core/messages/utils.py b/libs/core/langchain_core/messages/utils.pyindex aa3fe27dd5..45f1704614 100644--- a/libs/core/langchain_core/messages/utils.py+++ b/libs/core/langchain_core/messages/utils.py@@ -1191,6 +1191,8 @@ def convert_to_openai_messages(},})+ elif block.get("type") == "thinking":+ content.append(block)else:err = (f"Unrecognized content block at "diff --git a/libs/core/tests/unit_tests/messages/test_utils.py b/libs/core/tests/unit_tests/messages/test_utils.pyindex 8777b2674c..3b223b4864 100644--- a/libs/core/tests/unit_tests/messages/test_utils.py+++ b/libs/core/tests/unit_tests/messages/test_utils.py@@ -832,6 +832,18 @@ def test_convert_to_openai_messages_anthropic() -> None:]assert result == expected+ # Test thinking blocks (pass through)+ thinking_block = {+ "signature": "abc123",+ "thinking": "Thinking text.",+ "type": "thinking",+ }+ text_block = {"text": "Response text.", "type": "text"}+ messages = [AIMessage([thinking_block, text_block])]+ result = convert_to_openai_messages(messages)+ expected = [{"role": "assistant", "content": [thinking_block, text_block]}]+ assert result == expected+def test_convert_to_openai_messages_bedrock_converse_image() -> None:image_data = create_image_data()diff --git a/libs/partners/anthropic/tests/integration_tests/test_chat_models.py b/libs/partners/anthropic/tests/integration_tests/test_chat_models.pyindex cde69d25bc..c61817972f 100644--- a/libs/partners/anthropic/tests/integration_tests/test_chat_models.py+++ b/libs/partners/anthropic/tests/integration_tests/test_chat_models.py@@ -4,6 +4,7 @@ import jsonfrom base64 import b64encodefrom typing import List, Optional+import httpximport pytestimport requestsfrom anthropic import BadRequestError@@ -768,3 +769,64 @@ def test_structured_output_thinking_force_tool_use() -> None:)with pytest.raises(BadRequestError):llm.invoke("Generate a username for Sally with green hair")+++def test_image_tool_calling() -> None:+ """Test tool calling with image inputs."""++ class color_picker(BaseModel):+ """Input your fav color and get a random fact about it."""++ fav_color: str++ human_content: List[dict] = [+ {+ "type": "text",+ "text": "what's your favorite color in this image",+ },+ ]+ image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"+ image_data = b64encode(httpx.get(image_url).content).decode("utf-8")+ human_content.append(+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/jpeg",+ "data": image_data,+ },+ }+ )+ messages = [+ SystemMessage("you're a good assistant"),+ HumanMessage(human_content), # type: ignore[arg-type]+ AIMessage(+ [+ {"type": "text", "text": "Hmm let me think about that"},+ {+ "type": "tool_use",+ "input": {"fav_color": "green"},+ "id": "foo",+ "name": "color_picker",+ },+ ]+ ),+ HumanMessage(+ [+ {+ "type": "tool_result",+ "tool_use_id": "foo",+ "content": [+ {+ "type": "text",+ "text": "green is a great pick! that's my sister's favorite color", # noqa: E501+ }+ ],+ "is_error": False,+ },+ {"type": "text", "text": "what's my sister's favorite color"},+ ]+ ),+ ]+ llm = ChatAnthropic(model="claude-3-5-sonnet-latest")+ llm.bind_tools([color_picker]).invoke(messages)diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.pyindex e40c5bf98d..7cc4b2d9cf 100644--- a/libs/partners/openai/langchain_openai/chat_models/base.py+++ b/libs/partners/openai/langchain_openai/chat_models/base.py@@ -186,15 +186,38 @@ def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:def _format_message_content(content: Any) -> Any:"""Format message content."""if content and isinstance(content, list):- # Remove unexpected block typesformatted_content = []for block in content:+ # Remove unexpected block typesif (isinstance(block, dict)and "type" in block- and block["type"] == "tool_use"+ and block["type"] in ("tool_use", "thinking")):continue+ # Anthropic image blocks+ elif (+ isinstance(block, dict)+ and block.get("type") == "image"+ and (source := block.get("source"))+ and isinstance(source, dict)+ ):+ if source.get("type") == "base64" and (+ (media_type := source.get("media_type"))+ and (data := source.get("data"))+ ):+ formatted_content.append(+ {+ "type": "image_url",+ "image_url": {"url": f"data:{media_type};base64,{data}"},+ }+ )+ elif source.get("type") == "url" and (url := source.get("url")):+ formatted_content.append(+ {"type": "image_url", "image_url": {"url": url}}+ )+ else:+ continueelse:formatted_content.append(block)else:diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.pyindex f131dbaa2b..53528808c2 100644--- a/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base_standard.py@@ -29,6 +29,10 @@ class TestOpenAIStandard(ChatModelIntegrationTests):def supports_json_mode(self) -> bool:return True+ @property+ def supports_anthropic_inputs(self) -> bool:+ return True+@propertydef supported_usage_metadata_details(self,diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.pyindex abbe9fdf27..533abc2da9 100644--- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py+++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py@@ -1960,7 +1960,7 @@ class ChatModelIntegrationTests(ChatModelTests):set the ``supports_anthropic_inputs`` property to False.""" # noqa: E501if not self.supports_anthropic_inputs:- return+ pytest.skip("Model does not explicitly support Anthropic inputs.")class color_picker(BaseModelV1):"""Input your fav color and get a random fact about it."""@@ -1998,26 +1998,55 @@ class ChatModelIntegrationTests(ChatModelTests):"id": "foo","name": "color_picker",},+ ],+ tool_calls=[+ {+ "name": "color_picker",+ "args": {"fav_color": "green"},+ "id": "foo",+ "type": "tool_call",+ }+ ],+ ),+ ToolMessage("That's a great pick!", tool_call_id="foo"),+ ]+ response = model.bind_tools([color_picker]).invoke(messages)+ assert isinstance(response, AIMessage)++ # Test thinking blocks+ messages = [+ HumanMessage(+ [+ {+ "type": "text",+ "text": "Hello",+ },+ ]+ ),+ AIMessage(+ [+ {+ "type": "thinking",+ "thinking": "I'm thinking...",+ "signature": "abc123",+ },+ {+ "type": "text",+ "text": "Hello, how are you?",+ },]),HumanMessage([{- "type": "tool_result",- "tool_use_id": "foo",- "content": [- {- "type": "text",- "text": "green is a great pick! that's my sister's favorite color", # noqa: E501- }- ],- "is_error": False,+ "type": "text",+ "text": "Well, thanks.",},- {"type": "text", "text": "what's my sister's favorite color"},]),]- model.bind_tools([color_picker]).invoke(messages)+ response = model.invoke(messages)+ assert isinstance(response, AIMessage)def test_tool_message_error_status(self, model: BaseChatModel, my_adder_tool: BaseTool