The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(fireworks): translate canonical multimodal content blocks for chat completions (#37090)
murugand2304e7a55f0bbf
diff --git a/libs/partners/fireworks/langchain_fireworks/chat_models.py b/libs/partners/fireworks/langchain_fireworks/chat_models.pyindex 05ef1be5f3..5a4b7d7e1e 100644--- a/libs/partners/fireworks/langchain_fireworks/chat_models.py+++ b/libs/partners/fireworks/langchain_fireworks/chat_models.py@@ -57,6 +57,10 @@ from langchain_core.messages import (ToolCall,ToolMessage,ToolMessageChunk,+ is_data_content_block,+)+from langchain_core.messages.block_translators.openai import (+ convert_to_openai_data_block,)from langchain_core.messages.tool import (ToolCallChunk,@@ -166,6 +170,70 @@ def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:return ChatMessage(content=_dict.get("content", ""), role=role or "")+def _format_message_content(content: Any) -> Any:+ """Format message content for the Fireworks chat completions wire format.++ Adapted from `langchain_openai.chat_models.base._format_message_content`,+ scoped to the chat completions API: drops content block types the wire+ format does not carry, translates canonical v0/v1 multimodal data blocks+ via `convert_to_openai_data_block(block, api="chat/completions")`, and+ converts legacy Anthropic-shape image blocks (`{"type": "image",+ "source": {...}}`) to OpenAI `image_url` blocks. String and non-list+ content are returned unchanged.++ Args:+ content: The message content. Strings and non-list values are+ returned as-is; lists are walked block by block.++ Returns:+ The formatted content, ready to be placed on the chat completions+ wire. List inputs return a new list with translations applied; other+ inputs are returned unchanged.+ """+ if not isinstance(content, list):+ return content+ formatted: list[Any] = []+ for block in content:+ if isinstance(block, dict) and "type" in block:+ btype = block["type"]+ if btype in (+ "tool_use",+ "thinking",+ "reasoning_content",+ "function_call",+ "code_interpreter_call",+ ):+ continue+ if is_data_content_block(block):+ formatted.append(+ convert_to_openai_data_block(block, api="chat/completions")+ )+ continue+ if (+ btype == "image"+ and (source := block.get("source"))+ and isinstance(source, dict)+ ):+ if (+ source.get("type") == "base64"+ and (media_type := source.get("media_type"))+ and (data := source.get("data"))+ ):+ formatted.append(+ {+ "type": "image_url",+ "image_url": {"url": f"data:{media_type};base64,{data}"},+ }+ )+ continue+ if source.get("type") == "url" and (url := source.get("url")):+ formatted.append({"type": "image_url", "image_url": {"url": url}})+ continue+ continue+ formatted.append(block)+ return formatted++def _convert_message_to_dict(message: BaseMessage) -> dict:"""Convert a LangChain message to a dictionary.@@ -178,14 +246,23 @@ def _convert_message_to_dict(message: BaseMessage) -> dict:"""message_dict: dict[str, Any]if isinstance(message, ChatMessage):- message_dict = {"role": message.role, "content": message.content}+ message_dict = {+ "role": message.role,+ "content": _format_message_content(message.content),+ }elif isinstance(message, HumanMessage):- message_dict = {"role": "user", "content": message.content}+ message_dict = {+ "role": "user",+ "content": _format_message_content(message.content),+ }elif isinstance(message, AIMessage):# Translate v1 contentif message.response_metadata.get("output_version") == "v1":message = _convert_from_v1_to_chat_completions(message)- message_dict = {"role": "assistant", "content": message.content}+ message_dict = {+ "role": "assistant",+ "content": _format_message_content(message.content),+ }if "function_call" in message.additional_kwargs:message_dict["function_call"] = message.additional_kwargs["function_call"]# If function call only, content is None not empty string@@ -206,7 +283,10 @@ def _convert_message_to_dict(message: BaseMessage) -> dict:else:passelif isinstance(message, SystemMessage):- message_dict = {"role": "system", "content": message.content}+ message_dict = {+ "role": "system",+ "content": _format_message_content(message.content),+ }elif isinstance(message, FunctionMessage):message_dict = {"role": "function",@@ -216,7 +296,7 @@ def _convert_message_to_dict(message: BaseMessage) -> dict:elif isinstance(message, ToolMessage):message_dict = {"role": "tool",- "content": message.content,+ "content": _format_message_content(message.content),"tool_call_id": message.tool_call_id,}else:diff --git a/libs/partners/fireworks/tests/unit_tests/test_chat_models.py b/libs/partners/fireworks/tests/unit_tests/test_chat_models.pyindex 1e3d89d62d..23d0f1520e 100644--- a/libs/partners/fireworks/tests/unit_tests/test_chat_models.py+++ b/libs/partners/fireworks/tests/unit_tests/test_chat_models.py@@ -14,7 +14,14 @@ from fireworks.client.error import ( # type: ignore[import-untyped]RateLimitError,ServiceUnavailableError,)-from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage+from langchain_core.messages import (+ AIMessage,+ AIMessageChunk,+ ChatMessage,+ HumanMessage,+ SystemMessage,+ ToolMessage,+)from langchain_fireworks import ChatFireworksfrom langchain_fireworks.chat_models import (@@ -22,6 +29,8 @@ from langchain_fireworks.chat_models import (_completion_with_retry,_convert_chunk_to_message_chunk,_convert_dict_to_message,+ _convert_message_to_dict,+ _format_message_content,_usage_to_metadata,)@@ -92,6 +101,318 @@ def test_convert_dict_to_message_without_reasoning_content() -> None:assert "reasoning_content" not in message.additional_kwargs+def test_format_message_content_passthrough_string() -> None:+ """Plain string content is returned unchanged."""+ assert _format_message_content("hello") == "hello"+++def test_format_message_content_translates_v1_image_block() -> None:+ """Canonical v1 image block is translated to OpenAI image_url + data URI."""+ blocks = [{"type": "image", "base64": "abc", "mime_type": "image/png"}]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},+ ]+++def test_format_message_content_translates_v0_base64_image_block() -> None:+ """v0 source_type='base64' image block is translated."""+ blocks = [+ {+ "type": "image",+ "source_type": "base64",+ "data": "qqq",+ "mime_type": "image/png",+ }+ ]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,qqq"}},+ ]+++def test_format_message_content_passes_through_existing_image_url() -> None:+ """Already-OpenAI image_url blocks pass through unchanged."""+ blocks = [+ {"type": "image_url", "image_url": {"url": "https://example.com/y.png"}},+ ]++ formatted = _format_message_content(blocks)++ assert formatted == blocks+++ "btype",+ [+ "tool_use",+ "thinking",+ "reasoning_content",+ "function_call",+ "code_interpreter_call",+ ],+)+def test_format_message_content_drops_unsupported_block_types(btype: str) -> None:+ """Block types not part of the OpenAI chat completions wire format are stripped."""+ blocks = [+ {"type": "text", "text": "visible"},+ {"type": btype, "foo": "bar"},+ ]++ formatted = _format_message_content(blocks)++ assert formatted == [{"type": "text", "text": "visible"}]+++def test_format_message_content_preserves_order_around_dropped_blocks() -> None:+ """Surviving blocks keep their order when interleaved drops are removed."""+ blocks = [+ {"type": "text", "text": "before"},+ {"type": "thinking", "thinking": "..."},+ {"type": "image", "base64": "abc", "mime_type": "image/png"},+ {"type": "tool_use", "name": "t", "input": {}},+ {"type": "text", "text": "after"},+ ]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {"type": "text", "text": "before"},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},+ {"type": "text", "text": "after"},+ ]+++def test_format_message_content_translates_v1_url_image_block() -> None:+ """v1 image block with a top-level URL maps to an OpenAI image_url block."""+ blocks = [{"type": "image", "url": "https://example.com/img.png"}]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {"type": "image_url", "image_url": {"url": "https://example.com/img.png"}},+ ]+++def test_format_message_content_translates_v0_url_image_block() -> None:+ """v0 source_type=url image block is translated."""+ blocks = [+ {+ "type": "image",+ "source_type": "url",+ "url": "https://example.com/v0.png",+ }+ ]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {"type": "image_url", "image_url": {"url": "https://example.com/v0.png"}},+ ]+++def test_format_message_content_translates_anthropic_source_base64_image() -> None:+ """Legacy Anthropic-shape image with base64 source maps to a data URI."""+ blocks = [+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": "abc",+ },+ }+ ]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},+ ]+++def test_format_message_content_translates_anthropic_source_url_image() -> None:+ """Legacy Anthropic-shape image with url source maps to image_url."""+ blocks = [+ {+ "type": "image",+ "source": {"type": "url", "url": "https://example.com/anthropic.png"},+ }+ ]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/anthropic.png"},+ },+ ]+++def test_format_message_content_translates_v1_audio_block() -> None:+ """v1 audio block is translated to OpenAI input_audio shape."""+ blocks = [{"type": "audio", "base64": "aGVsbG8=", "mime_type": "audio/wav"}]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {"type": "input_audio", "input_audio": {"data": "aGVsbG8=", "format": "wav"}},+ ]+++def test_format_message_content_translates_v1_file_block_base64() -> None:+ """v1 file block with base64 + filename maps to OpenAI file_data shape."""+ blocks = [+ {+ "type": "file",+ "base64": "JVBERi0=",+ "mime_type": "application/pdf",+ "filename": "x.pdf",+ }+ ]++ formatted = _format_message_content(blocks)++ assert formatted == [+ {+ "type": "file",+ "file": {+ "file_data": "data:application/pdf;base64,JVBERi0=",+ "filename": "x.pdf",+ },+ },+ ]+++def test_convert_message_to_dict_translates_tool_message_image() -> None:+ """ToolMessage with a canonical image block lands as OpenAI image_url on the wire.++ Reproduces the failure mode where a tool that returns an image (e.g. a+ file-reader) hands back `content_blocks=[{"type": "image", ...}]` and the+ message round-trips into a Fireworks chat completions request.+ """+ tool_message = ToolMessage(+ content=[{"type": "image", "base64": "abc", "mime_type": "image/png"}],+ tool_call_id="call_1",+ )++ result = _convert_message_to_dict(tool_message)++ assert result == {+ "role": "tool",+ "content": [+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},+ ],+ "tool_call_id": "call_1",+ }+++def test_convert_message_to_dict_translates_human_mixed_content() -> None:+ """HumanMessage with mixed text + image blocks translates correctly."""+ human_message = HumanMessage(+ content=[+ {"type": "text", "text": "what is this?"},+ {"type": "image", "base64": "xyz", "mime_type": "image/jpeg"},+ ]+ )++ result = _convert_message_to_dict(human_message)++ assert result == {+ "role": "user",+ "content": [+ {"type": "text", "text": "what is this?"},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/jpeg;base64,xyz"},+ },+ ],+ }+++def test_convert_message_to_dict_chat_message_uses_translator() -> None:+ """ChatMessage path also runs content through the formatter."""+ chat_message = ChatMessage(+ role="custom",+ content=[{"type": "image", "base64": "zz", "mime_type": "image/gif"}],+ )++ result = _convert_message_to_dict(chat_message)++ assert result == {+ "role": "custom",+ "content": [+ {"type": "image_url", "image_url": {"url": "data:image/gif;base64,zz"}},+ ],+ }+++def test_convert_message_to_dict_string_content_unchanged() -> None:+ """String content on common message types passes through unmodified."""+ assert _convert_message_to_dict(HumanMessage(content="hi"))["content"] == "hi"+ assert _convert_message_to_dict(SystemMessage(content="sys"))["content"] == "sys"+ assert (+ _convert_message_to_dict(ToolMessage(content="r1", tool_call_id="t"))["content"]+ == "r1"+ )+++def test_convert_message_to_dict_translates_system_list_content() -> None:+ """SystemMessage with list content is routed through the formatter."""+ system_message = SystemMessage(+ content=[+ {"type": "text", "text": "rules"},+ {"type": "thinking", "thinking": "drop me"},+ ]+ )++ result = _convert_message_to_dict(system_message)++ assert result == {+ "role": "system",+ "content": [{"type": "text", "text": "rules"}],+ }+++def test_convert_message_to_dict_translates_ai_message_image_content() -> None:+ """AIMessage with a canonical image block is translated, not forwarded raw."""+ ai_message = AIMessage(+ content=[+ {"type": "text", "text": "see attached"},+ {"type": "image", "base64": "abc", "mime_type": "image/png"},+ ]+ )++ result = _convert_message_to_dict(ai_message)++ assert result == {+ "role": "assistant",+ "content": [+ {"type": "text", "text": "see attached"},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,abc"}},+ ],+ }+++def test_convert_message_to_dict_propagates_translator_value_error() -> None:+ """Translator errors surface to callers instead of shipping bad payloads.++ Chat completions does not support file URLs; the translator raises rather+ than letting an unsupported block through.+ """+ bad_message = HumanMessage(+ content=[{"type": "file", "url": "https://example.com/doc.pdf"}]+ )++ with pytest.raises(ValueError, match="file URLs"):+ _convert_message_to_dict(bad_message)++def _make_llm(max_retries: int | None = 2) -> ChatFireworks:return ChatFireworks(model="accounts/fireworks/models/test",