Skip to content

The agent engineering platform.

fix(fireworks): preserve reasoning in streaming and tool loops (#41093)

nm-fireworkse986d201b823
diff --git a/libs/partners/fireworks/langchain_fireworks/chat_models.py b/libs/partners/fireworks/langchain_fireworks/chat_models.py
index 6d58e2034e..3bb8469ea6 100644
--- a/libs/partners/fireworks/langchain_fireworks/chat_models.py
+++ b/libs/partners/fireworks/langchain_fireworks/chat_models.py
@@ -155,7 +155,10 @@ def _convert_dict_to_message(_dict: Mapping[str, Any]) -> BaseMessage:
# Also Fireworks returns None for tool invocations
content = _dict.get("content", "") or ""
additional_kwargs: dict = {}
- if reasoning_content := _dict.get("reasoning_content"):
+ if (
+ isinstance(reasoning_content := _dict.get("reasoning_content"), str)
+ and reasoning_content
+ ):
additional_kwargs["reasoning_content"] = reasoning_content
if function_call := _dict.get("function_call"):
@@ -398,6 +401,13 @@ def _convert_message_to_dict(message: BaseMessage) -> dict:
_format_message_content(message.content)
),
}
+ # Replay reasoning only when the message is tagged `fireworks` or has no
+ # `model_provider` (hand-built messages, or integrations that don't tag
+ # theirs); reasoning from other tagged providers may not be valid input.
+ if isinstance(
+ reasoning_content := message.additional_kwargs.get("reasoning_content"), str
+ ) and message.response_metadata.get("model_provider") in (None, "fireworks"):
+ message_dict["reasoning_content"] = reasoning_content
if "function_call" in message.additional_kwargs:
message_dict["function_call"] = message.additional_kwargs["function_call"]
# If function call only, content is None not empty string
@@ -470,6 +480,11 @@ def _usage_to_metadata(usage: Mapping[str, Any]) -> UsageMetadata:
cached_tokens = (usage.get("prompt_tokens_details") or {}).get("cached_tokens")
if cached_tokens is not None:
usage_metadata["input_token_details"] = {"cache_read": cached_tokens}
+ reasoning_tokens = (usage.get("completion_tokens_details") or {}).get(
+ "reasoning_tokens"
+ )
+ if reasoning_tokens is not None:
+ usage_metadata["output_token_details"] = {"reasoning": reasoning_tokens}
return usage_metadata
@@ -552,6 +567,11 @@ def _convert_chunk_to_message_chunk(
content = cast(str, _dict.get("content") or "")
additional_kwargs: dict = {}
tool_call_chunks: list[ToolCallChunk] = []
+ if (
+ isinstance(reasoning_content := _dict.get("reasoning_content"), str)
+ and reasoning_content
+ ):
+ additional_kwargs["reasoning_content"] = reasoning_content
if _dict.get("function_call"):
function_call = dict(_dict["function_call"])
if "name" in function_call and function_call["name"] is None:
@@ -592,6 +612,67 @@ def _convert_chunk_to_message_chunk(
return default_class(content=content) # type: ignore[call-arg]
+def _format_chunk_for_v1(message_chunk: AIMessageChunk) -> AIMessageChunk:
+ """Convert a stream chunk to v1 content blocks with separate index namespaces.
+
+ Fireworks streams a single text field and a single `reasoning_content` field,
+ but numbers tool calls from 0. Left to core, streaming (`stream`, `astream`,
+ and streaming `invoke`) gives unindexed blocks an integer that increments each
+ time the block type changes, so tool call 0 would be merged into the reasoning
+ block by `merge_lists`, and reasoning interrupted by text would split in two.
+
+ `merge_lists` merges blocks with matching indices only when the index is an int
+ or a string starting with `lc_`; other strings are appended unmerged. The raw
+ `tool_call_chunks` keep the provider's integer indices, which
+ `AIMessageChunk.__add__` relies on to merge tool-call arguments.
+ """
+ blocks: list[dict[str, Any]] = []
+ if reasoning := message_chunk.additional_kwargs.get("reasoning_content"):
+ blocks.append(
+ {"type": "reasoning", "reasoning": reasoning, "index": "lc_reasoning"}
+ )
+ if message_chunk.content:
+ blocks.append(
+ {"type": "text", "text": message_chunk.content, "index": "lc_text"}
+ )
+ for tool_call_chunk in message_chunk.tool_call_chunks:
+ block: dict[str, Any] = {
+ "type": "tool_call_chunk",
+ "id": tool_call_chunk["id"],
+ "name": tool_call_chunk["name"],
+ "args": tool_call_chunk["args"],
+ }
+ if (index := tool_call_chunk["index"]) is not None:
+ block["index"] = f"lc_tc_{index}"
+ blocks.append(block)
+ return message_chunk.model_copy(
+ update={
+ "content": blocks,
+ "response_metadata": {
+ **message_chunk.response_metadata,
+ "output_version": "v1",
+ },
+ }
+ )
+
+
+def _finalize_stream(
+ stream: Iterator[ChatGenerationChunk],
+) -> Iterator[ChatGenerationChunk]:
+ """Supply core's finalization marker when aggregating v1 chunks internally."""
+ yield from stream
+ yield ChatGenerationChunk(message=AIMessageChunk(content="", chunk_position="last"))
+
+
+async def _afinalize_stream(
+ stream: AsyncIterator[ChatGenerationChunk],
+) -> AsyncIterator[ChatGenerationChunk]:
+ """Supply core's finalization marker when aggregating v1 chunks internally."""
+ async for chunk in stream:
+ yield chunk
+ yield ChatGenerationChunk(message=AIMessageChunk(content="", chunk_position="last"))
+
+
class _RetryableHTTPStatusError(FireworksError):
"""Internal marker for 5xx `httpx.HTTPStatusError` responses.
@@ -1288,6 +1369,10 @@ class ChatFireworks(BaseChatModel):
if not isinstance(chunk, dict):
chunk = chunk.model_dump()
message_chunk = _convert_chunk_to_message_chunk(chunk, default_chunk_class)
+ if self.output_version == "v1" and isinstance(
+ message_chunk, AIMessageChunk
+ ):
+ message_chunk = _format_chunk_for_v1(message_chunk)
generation_info: dict[str, Any] = {}
logprobs = None
if choices := chunk.get("choices"):
@@ -1323,6 +1408,8 @@ class ChatFireworks(BaseChatModel):
stream_iter = self._stream(
messages, stop=stop, run_manager=run_manager, **kwargs
)
+ if self.output_version == "v1":
+ stream_iter = _finalize_stream(stream_iter)
return generate_from_stream(stream_iter)
message_dicts, params = self._create_message_dicts(messages, stop)
params = {
@@ -1405,6 +1492,10 @@ class ChatFireworks(BaseChatModel):
if not isinstance(chunk, dict):
chunk = chunk.model_dump()
message_chunk = _convert_chunk_to_message_chunk(chunk, default_chunk_class)
+ if self.output_version == "v1" and isinstance(
+ message_chunk, AIMessageChunk
+ ):
+ message_chunk = _format_chunk_for_v1(message_chunk)
generation_info: dict[str, Any] = {}
logprobs = None
if choices := chunk.get("choices"):
@@ -1442,6 +1533,8 @@ class ChatFireworks(BaseChatModel):
stream_iter = self._astream(
messages, stop=stop, run_manager=run_manager, **kwargs
)
+ if self.output_version == "v1":
+ stream_iter = _afinalize_stream(stream_iter)
return await agenerate_from_stream(stream_iter)
message_dicts, params = self._create_message_dicts(messages, stop)
diff --git a/libs/partners/fireworks/tests/unit_tests/test_reasoning.py b/libs/partners/fireworks/tests/unit_tests/test_reasoning.py
new file mode 100644
index 0000000000..a922b3b4ce
--- /dev/null
+++ b/libs/partners/fireworks/tests/unit_tests/test_reasoning.py
@@ -0,0 +1,436 @@
+"""Tests for `ChatFireworks` reasoning preservation, usage details, and v1 streaming."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncIterator, Iterator
+from typing import Any, Literal
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+from fireworks import AsyncFireworks, Fireworks
+from langchain_core.messages import (
+ AIMessage,
+ AIMessageChunk,
+ BaseMessage,
+ HumanMessage,
+ ToolMessage,
+ message_chunk_to_message,
+)
+
+from langchain_fireworks import ChatFireworks
+from langchain_fireworks.chat_models import (
+ _convert_chunk_to_message_chunk,
+ _convert_dict_to_message,
+ _convert_message_to_dict,
+ _usage_to_metadata,
+)
+
+
+def _tool_stream() -> Iterator[dict[str, Any]]:
+ yield {"choices": [{"delta": {"role": "assistant", "reasoning_content": "Need "}}]}
+ yield {"choices": [{"delta": {"reasoning_content": "a lookup."}}]}
+ yield {
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "id": "call_lookup",
+ "type": "function",
+ "function": {
+ "name": "lookup",
+ "arguments": '{"query":"example"}',
+ },
+ }
+ ]
+ },
+ "finish_reason": "tool_calls",
+ }
+ ]
+ }
+ yield {
+ "choices": [],
+ "usage": {
+ "prompt_tokens": 20,
+ "completion_tokens": 15,
+ "total_tokens": 35,
+ "completion_tokens_details": {"reasoning_tokens": 11},
+ },
+ }
+
+
+async def _async_tool_stream() -> AsyncIterator[dict[str, Any]]:
+ for chunk in _tool_stream():
+ yield chunk
+
+
[email protected]("reasoning", ["Let me think.", "", None])
+def test_assistant_reasoning_round_trip(reasoning: str | None) -> None:
+ message = AIMessage(
+ content="Answer", additional_kwargs={"reasoning_content": reasoning}
+ )
+ result = _convert_message_to_dict(message)
+ if reasoning is None:
+ assert "reasoning_content" not in result
+ else:
+ assert result["reasoning_content"] == reasoning
+ assert result["content"] == "Answer"
+
+
+def test_non_streaming_reasoning_round_trip_with_tool_call() -> None:
+ raw = {
+ "role": "assistant",
+ "content": None,
+ "reasoning_content": "Need a lookup.",
+ "tool_calls": [
+ {
+ "id": "call_lookup",
+ "type": "function",
+ "function": {"name": "lookup", "arguments": '{"query": "example"}'},
+ }
+ ],
+ }
+ assert _convert_message_to_dict(_convert_dict_to_message(raw)) == raw
+
+
+def test_streamed_reasoning_accumulates_and_replays() -> None:
+ chunks = [
+ _convert_chunk_to_message_chunk(chunk, AIMessageChunk)
+ for chunk in _tool_stream()
+ ]
+ message = message_chunk_to_message(sum(chunks[1:], chunks[0]))
+ assert isinstance(message, AIMessage)
+ result = _convert_message_to_dict(message)
+ assert result["reasoning_content"] == "Need a lookup."
+ assert result["content"] is None
+ assert result["tool_calls"][0]["function"]["name"] == "lookup"
+ assert message.usage_metadata == {
+ "input_tokens": 20,
+ "output_tokens": 15,
+ "total_tokens": 35,
+ "output_token_details": {"reasoning": 11},
+ }
+
+
+ "delta", [{}, {"reasoning_content": None}, {"reasoning_content": ""}]
+)
+def test_stream_without_reasoning_does_not_add_field(delta: dict[str, Any]) -> None:
+ chunk = _convert_chunk_to_message_chunk(
+ {"choices": [{"delta": {"content": "Answer", **delta}}]}, AIMessageChunk
+ )
+ assert "reasoning_content" not in chunk.additional_kwargs
+ assert "reasoning_content" not in _convert_message_to_dict(
+ message_chunk_to_message(chunk)
+ )
+
+
+def test_replay_keeps_reasoning_out_of_content_blocks() -> None:
+ message = AIMessage(
+ content=[
+ {"type": "reasoning", "reasoning": "Foreign provider reasoning"},
+ {"type": "text", "text": "Answer", "index": 0},
+ ],
+ additional_kwargs={"reasoning_content": "Fireworks reasoning"},
+ response_metadata={"output_version": "v1"},
+ )
+ result = _convert_message_to_dict(message)
+ assert result["content"] == "Answer"
+ assert result["reasoning_content"] == "Fireworks reasoning"
+
+
[email protected]("reasoning_tokens", [11, 0])
+def test_usage_reasoning_is_a_detail_not_extra_output(reasoning_tokens: int) -> None:
+ assert _usage_to_metadata(
+ {
+ "prompt_tokens": 20,
+ "completion_tokens": 15,
+ "prompt_tokens_details": {"cached_tokens": 7},
+ "completion_tokens_details": {"reasoning_tokens": reasoning_tokens},
+ }
+ ) == {
+ "input_tokens": 20,
+ "output_tokens": 15,
+ "total_tokens": 35,
+ "input_token_details": {"cache_read": 7},
+ "output_token_details": {"reasoning": reasoning_tokens},
+ }
+
+
[email protected]("details", [None, {}, {"reasoning_tokens": None}])
+def test_usage_without_reasoning_omits_detail(details: dict[str, Any] | None) -> None:
+ assert "output_token_details" not in _usage_to_metadata(
+ {"completion_tokens_details": details}
+ )
+
+
[email protected]("output_version", ["v0", "v1"])
[email protected]("method", ["stream", "invoke", "astream", "ainvoke"])
[email protected]("disable_streaming", [False, True, "tool_calling"])
+async def test_reasoning_survives_two_tool_turns(
+ method: str,
+ output_version: Literal["v0", "v1"],
+ *,
+ disable_streaming: bool | Literal["tool_calling"],
+) -> None:
+ """Exercise chunk assembly and request serialization against a mocked client."""
+ client = MagicMock()
+ async_client = MagicMock()
+ client.create.side_effect = lambda **_: _tool_stream()
+ async_client.create = AsyncMock(side_effect=lambda **_: _async_tool_stream())
+ chat_model = ChatFireworks(
+ model="accounts/fireworks/models/test-model",
+ api_key="fake-key", # type: ignore[arg-type]
+ streaming=True,
+ disable_streaming=disable_streaming,
+ output_version=output_version,
+ client=client,
+ async_client=async_client,
+ )
+ model = chat_model.bind_tools(
+ [{"name": "lookup", "description": "Look up a value", "parameters": {}}]
+ )
+ history: list[BaseMessage] = [HumanMessage(content="Find a value")]
+ for _ in range(2):
+ if method == "stream":
+ chunks = list(model.stream(history))
+ response = message_chunk_to_message(sum(chunks[1:], chunks[0]))
+ elif method == "astream":
+ chunks = [chunk async for chunk in model.astream(history)]
+ response = message_chunk_to_message(sum(chunks[1:], chunks[0]))
+ elif method == "invoke":
+ response = model.invoke(history)
+ else:
+ response = await model.ainvoke(history)
+ assert isinstance(response, AIMessage)
+ assert response.additional_kwargs["reasoning_content"] == "Need a lookup."
+ assert [block["type"] for block in response.content_blocks] == [
+ "reasoning",
+ "tool_call",
+ ]
+ assert response.content_blocks[0].get("reasoning") == "Need a lookup."
+ assert response.content_blocks[1].get("args") == {"query": "example"}
+ assert response.usage_metadata is not None
+ assert response.usage_metadata["output_token_details"] == {"reasoning": 11}
+ history.extend(
+ [response, ToolMessage(content="A value", tool_call_id="call_lookup")]
+ )
+ client_mock = async_client.create if method.startswith("a") else client.create
+ sent = client_mock.call_args_list[1].kwargs["messages"]
+ assert sent[1]["reasoning_content"] == "Need a lookup."
+ assert sent[1]["content"] in (None, [])
+ assert sent[1]["tool_calls"][0]["id"] == "call_lookup"
+ assert sent[2] == {
+ "role": "tool",
+ "content": "A value",
+ "tool_call_id": "call_lookup",
+ }
+ replayed, _ = chat_model._create_message_dicts(history, stop=None)
+ assert [replayed[index]["reasoning_content"] for index in (1, 3)] == [
+ "Need a lookup.",
+ "Need a lookup.",
+ ]
+
+
[email protected]("async_mode", [False, True])
+async def test_v1_stream_separates_reasoning_text_and_parallel_tool_calls(
+ *, async_mode: bool
+) -> None:
+ """Keep canonical blocks separate while merging fragmented tool arguments."""
+ deltas: list[dict[str, Any]] = [
+ {"role": "assistant", "reasoning_content": "Need ", "content": "Looking "},
+ {"content": "up.", "reasoning_content": ""},
+ {
+ "tool_calls": [
+ {
+ "index": index,
+ "id": f"call_{index}",
+ "type": "function",
+ "function": {"name": "lookup", "arguments": '{"query":"'},
+ }
+ for index in (0, 1)
+ ]
+ },
+ {
+ "reasoning_content": "two lookups.",
+ "tool_calls": [
+ {"index": 0, "function": {"arguments": 'first"}'}},
+ {"index": 1, "function": {"arguments": 'second"}'}},
+ ],
+ },
+ ]
+ raw_chunks: list[dict[str, Any]] = [
+ {"choices": [{"delta": delta}]} for delta in deltas
+ ]
+ raw_chunks.append({"choices": [{"delta": {}, "finish_reason": "tool_calls"}]})
+
+ async def async_chunks() -> AsyncIterator[dict[str, Any]]:
+ for chunk in raw_chunks:
+ yield chunk
+
+ client = MagicMock()
+ client.create.return_value = iter(raw_chunks)
+ async_client = MagicMock()
+ async_client.create = AsyncMock(return_value=async_chunks())
+ model = ChatFireworks(
+ model="accounts/fireworks/models/test-model",
+ api_key="fake-key", # type: ignore[arg-type]
+ client=client,
+ async_client=async_client,
+ output_version="v1",
+ )
+ chunks = (
+ [chunk async for chunk in model.astream("Find two values")]
+ if async_mode
+ else list(model.stream("Find two values"))
+ )
+ response = sum(chunks[1:], chunks[0])
+ blocks = response.content_blocks
+ assert [block["type"] for block in blocks] == [
+ "reasoning",
+ "text",
+ "tool_call",
+ "tool_call",
+ ]
+ assert blocks[0].get("reasoning") == "Need two lookups."
+ assert blocks[1].get("text") == "Looking up."
+ assert [block.get("args") for block in blocks[2:]] == [
+ {"query": "first"},
+ {"query": "second"},
+ ]
+ assert [tool_call["id"] for tool_call in response.tool_calls] == [
+ "call_0",
+ "call_1",
+ ]
+ assert {
+ tool_chunk["index"] for chunk in chunks for tool_chunk in chunk.tool_call_chunks
+ } == {0, 1}
+ assert (
+ len({block.get("index") for chunk in chunks for block in chunk.content_blocks})
+ == 4
+ )
+
+
[email protected]("provider", ["deepseek", "groq", "ollama", "xai"])
+def test_foreign_provider_reasoning_is_not_replayed(provider: str) -> None:
+ message = AIMessage(
+ content="Answer",
+ additional_kwargs={"reasoning_content": "Foreign reasoning"},
+ response_metadata={"model_provider": provider},
+ )
+ assert "reasoning_content" not in _convert_message_to_dict(message)
+
+
+ "reasoning", [123, False, {"text": "Reasoning"}, ["Reasoning"]]
+)
+def test_non_string_reasoning_is_not_replayed(reasoning: Any) -> None:
+ message = AIMessage(
+ content="Answer", additional_kwargs={"reasoning_content": reasoning}
+ )
+ assert "reasoning_content" not in _convert_message_to_dict(message)
+
+
[email protected]("output_version", ["v0", "v1"])
[email protected]("streaming", [False, True])
[email protected]("async_mode", [False, True])
+async def test_reasoning_round_trip_through_sdk_http_transport(
+ output_version: Literal["v0", "v1"], *, streaming: bool, async_mode: bool
+) -> None:
+ """Check SDK decoding and actual HTTP request serialization without sockets."""
+ requests: list[dict[str, Any]] = []
+
+ def respond(request: httpx.Request) -> httpx.Response:
+ requests.append(json.loads(request.content))
+ chunks = list(_tool_stream())
+ common = {"id": "completion_test", "model": "test-model", "created": 0}
+ if streaming:
+ events = [
+ "data: "
+ + json.dumps({**common, "object": "chat.completion.chunk", **chunk})
+ + "\n\n"
+ for chunk in chunks
+ ]
+ return httpx.Response(
+ 200,
+ headers={"content-type": "text/event-stream"},
+ content="".join([*events, "data: [DONE]\n\n"]),
+ )
+ tool_calls = chunks[2]["choices"][0]["delta"]["tool_calls"]
+ return httpx.Response(
+ 200,
+ json={
+ **common,
+ "object": "chat.completion",
+ "choices": [
+ {
+ "index": 0,
+ "finish_reason": "tool_calls",
+ "message": {
+ "role": "assistant",
+ "content": None,
+ "reasoning_content": "Need a lookup.",
+ "tool_calls": tool_calls,
+ },
+ }
+ ],
+ "usage": chunks[-1]["usage"],
+ },
+ )
+
+ transport = httpx.MockTransport(respond)
+ with Fireworks(
+ api_key="fake-key", http_client=httpx.Client(transport=transport), max_retries=0
+ ) as sdk:
+ async with AsyncFireworks(
+ api_key="fake-key",
+ http_client=httpx.AsyncClient(transport=transport),
+ max_retries=0,
+ ) as async_sdk:
+ model = ChatFireworks(
+ model="accounts/fireworks/models/test-model",
+ api_key="fake-key", # type: ignore[arg-type]
+ client=sdk.chat.completions,
+ async_client=async_sdk.chat.completions,
+ streaming=streaming,
+ output_version=output_version,
+ ).bind_tools(
+ [{"name": "lookup", "description": "Look up a value", "parameters": {}}]
+ )
+ history: list[BaseMessage] = [HumanMessage(content="Find a value")]
+ for _ in range(2):
+ response = (
+ await model.ainvoke(history)
+ if async_mode
+ else model.invoke(history)
+ )
+ assert isinstance(response, AIMessage)
+ assert (
+ response.additional_kwargs["reasoning_content"] == "Need a lookup."
+ )
+ assert [block["type"] for block in response.content_blocks] == [
+ "reasoning",
+ "tool_call",
+ ]
+ assert response.content_blocks[0].get("reasoning") == "Need a lookup."
+ assert response.content_blocks[1].get("args") == {"query": "example"}
+ assert response.usage_metadata == {
+ "input_tokens": 20,
+ "output_tokens": 15,
+ "total_tokens": 35,
+ "output_token_details": {"reasoning": 11},
+ }
+ history.extend(
+ [
+ response,
+ ToolMessage(content="A value", tool_call_id="call_lookup"),
+ ]
+ )
+ assert len(requests) == 2
+ assert requests[1]["messages"][1]["reasoning_content"] == "Need a lookup."
+ assert requests[1]["messages"][1]["tool_calls"][0]["id"] == "call_lookup"
+ assert requests[1]["messages"][2]["role"] == "tool"
+ assert requests[1]["messages"][2]["tool_call_id"] == "call_lookup"