Skip to content

A programming framework for agentic AI

Emit <think> and </think> around reasoning chunks from model_extras in choices.detla

y26s4824k2640bec835d598c
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
index 4d3b1a5b4..efdb22cd1 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py
@@ -69,7 +69,6 @@ from openai.types.chat import (
completion_create_params,
)
from openai.types.chat.chat_completion import Choice
-from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
from openai.types.shared_params import (
FunctionDefinition,
FunctionParameters,
@@ -724,21 +723,19 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
)
# Prepare data to process streaming chunks.
- choice: Union[ParsedChoice[Any], ParsedChoice[BaseModel], ChunkChoice] = cast(ChunkChoice, None)
- chunk = None
+ chunk: ChatCompletionChunk | None = None
stop_reason = None
maybe_model = None
content_deltas: List[str] = []
thought_deltas: List[str] = []
full_tool_calls: Dict[int, FunctionCall] = {}
- completion_tokens = 0
logprobs: Optional[List[ChatCompletionTokenLogprob]] = None
empty_chunk_warning_has_been_issued: bool = False
empty_chunk_warning_threshold: int = 10
empty_chunk_count = 0
-
first_chunk = True
+ is_reasoning = False
# Process the stream of chunks.
async for chunk in chunks:
@@ -750,6 +747,10 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
messages=cast(List[Dict[str, Any]], create_params.messages),
)
)
+
+ # Set the model from the lastest chunk.
+ maybe_model = chunk.model
+
# Empty chunks has been observed when the endpoint is under heavy load.
# https://github.com/microsoft/autogen/issues/4213
if len(chunk.choices) == 0:
@@ -764,21 +765,41 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
else:
empty_chunk_count = 0
- # to process usage chunk in streaming situations
- # add stream_options={"include_usage": True} in the initialization of OpenAIChatCompletionClient(...)
- # However the different api's
- # OPENAI api usage chunk produces no choices so need to check if there is a choice
- # liteLLM api usage chunk does produce choices
- choice = (
- chunk.choices[0]
- if len(chunk.choices) > 0
- else (choice if chunk.usage is not None and stop_reason is not None else cast(ChunkChoice, None))
- )
+ if len(chunk.choices) > 1:
+ # This is a multi-choice chunk, we need to warn the user.
+ warnings.warn(
+ f"Received a chunk with {len(chunk.choices)} choices. Only the first choice will be used.",
+ UserWarning,
+ stacklevel=2,
+ )
+
+ # Set the choice to the first choice in the chunk.
+ choice = chunk.choices[0]
# for liteLLM chunk usage, do the following hack keeping the pervious chunk.stop_reason (if set).
# set the stop_reason for the usage chunk to the prior stop_reason
stop_reason = choice.finish_reason if chunk.usage is None and stop_reason is None else stop_reason
maybe_model = chunk.model
+
+ reasoning_content: str | None = None
+ if choice.delta.model_extra is not None and "reasoning_content" in choice.delta.model_extra:
+ # If there is a reasoning_content field, then we populate the thought field. This is for models such as R1.
+ reasoning_content = choice.delta.model_extra.get("reasoning_content")
+
+ if isinstance(reasoning_content, str) and len(reasoning_content) > 0:
+ if not is_reasoning:
+ # Enter reasoning mode.
+ reasoning_content = "<think>" + reasoning_content
+ is_reasoning = True
+ thought_deltas.append(reasoning_content)
+ yield reasoning_content
+ elif is_reasoning:
+ # Exit reasoning mode.
+ reasoning_content = "</think>"
+ thought_deltas.append(reasoning_content)
+ is_reasoning = False
+ yield reasoning_content
+
# First try get content
if choice.delta.content:
content_deltas.append(choice.delta.content)
@@ -787,12 +808,6 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
# NOTE: for OpenAI, tool_calls and content are mutually exclusive it seems, so we can skip the rest of the loop.
# However, this may not be the case for other APIs -- we should expect this may need to be updated.
continue
- # if there is a reasoning_content field, then we populate the thought field. This is for models such as R1.
- if choice.delta.model_extra is not None:
- reasoning_content = choice.delta.model_extra.get("reasoning_content")
- if reasoning_content is not None:
- thought_deltas.append(reasoning_content)
- yield reasoning_content
# Otherwise, get tool calls
if choice.delta.tool_calls is not None:
for tool_call_chunk in choice.delta.tool_calls:
@@ -863,13 +878,13 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
)
content = ""
- # Always set thoughts if we have any, regardless of other content types
- if thought_deltas:
- thought = "".join(thought_deltas)
+ # Set thoughts if we have any reasoning content.
+ if thought_deltas:
+ thought = "".join(thought_deltas).lstrip("<think>").rstrip("</think>")
- # This is for local R1 models.
- if isinstance(content, str) and self._model_info["family"] == ModelFamily.R1 and thought is None:
- thought, content = parse_r1_content(content)
+ # This is for local R1 models whose reasoning content is within the content string.
+ if isinstance(content, str) and self._model_info["family"] == ModelFamily.R1 and thought is None:
+ thought, content = parse_r1_content(content)
# Create the result.
result = CreateResult(
diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.py
index 6fcc6dc3b..7ae6aeee0 100644
--- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py
+++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py
@@ -1090,13 +1090,14 @@ async def test_r1_reasoning_content_streaming(monkeypatch: pytest.MonkeyPatch) -
# Verify that the chunks first stream the reasoning content and then the main content
# Then verify that the final result has the correct content and thought
- assert len(chunks) == 4
- assert chunks[0] == "This is the reasoning content 1"
+ assert len(chunks) == 5
+ assert chunks[0] == "<think>This is the reasoning content 1"
assert chunks[1] == "This is the reasoning content 2"
- assert chunks[2] == "This is the main content"
- assert isinstance(chunks[3], CreateResult)
- assert chunks[3].content == "This is the main content"
- assert chunks[3].thought == "This is the reasoning content 1This is the reasoning content 2"
+ assert chunks[2] == "</think>"
+ assert chunks[3] == "This is the main content"
+ assert isinstance(chunks[4], CreateResult)
+ assert chunks[4].content == "This is the main content"
+ assert chunks[4].thought == "This is the reasoning content 1This is the reasoning content 2"
@pytest.mark.asyncio
@@ -1783,7 +1784,10 @@ async def test_openai_structured_output_with_streaming(model: str, openai_client
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model",
- ["gpt-4o-mini", "gemini-1.5-flash"],
+ [
+ "gpt-4o-mini",
+ # "gemini-1.5-flash", # Gemini models do not support structured output with tool calls from model client.
+ ],
)
async def test_openai_structured_output_with_tool_calls(model: str, openai_client: OpenAIChatCompletionClient) -> None:
class AgentResponse(BaseModel):
@@ -1796,13 +1800,15 @@ async def test_openai_structured_output_with_tool_calls(model: str, openai_clien
tool = FunctionTool(sentiment_analysis, description="Sentiment Analysis", strict=True)
+ extra_create_args = {"tool_choice": "required"}
+
response1 = await openai_client.create(
messages=[
SystemMessage(content="Analyze input text sentiment using the tool provided."),
UserMessage(content="I am happy.", source="user"),
],
tools=[tool],
- extra_create_args={"tool_choice": "required"},
+ extra_create_args=extra_create_args,
json_output=AgentResponse,
)
assert isinstance(response1.content, list)
@@ -1836,7 +1842,10 @@ async def test_openai_structured_output_with_tool_calls(model: str, openai_clien
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model",
- ["gpt-4o-mini", "gemini-1.5-flash"],
+ [
+ "gpt-4o-mini",
+ # "gemini-1.5-flash", # Gemini models do not support structured output with tool calls from model client.
+ ],
)
async def test_openai_structured_output_with_streaming_tool_calls(
model: str, openai_client: OpenAIChatCompletionClient
@@ -1851,6 +1860,8 @@ async def test_openai_structured_output_with_streaming_tool_calls(
tool = FunctionTool(sentiment_analysis, description="Sentiment Analysis", strict=True)
+ extra_create_args = {"tool_choice": "required"}
+
chunks1: List[str | CreateResult] = []
stream1 = openai_client.create_stream(
messages=[
@@ -1858,7 +1869,7 @@ async def test_openai_structured_output_with_streaming_tool_calls(
UserMessage(content="I am happy.", source="user"),
],
tools=[tool],
- extra_create_args={"tool_choice": "required"},
+ extra_create_args=extra_create_args,
json_output=AgentResponse,
)
async for chunk in stream1: