A programming framework for agentic AI
Emit <think> and </think> around reasoning chunks from model_extras in choices.detla
y26s4824k2640bec835d598c
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.pyindex 4d3b1a5b4..efdb22cd1 100644--- a/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py+++ b/python/packages/autogen-ext/src/autogen_ext/models/openai/_openai_client.py@@ -69,7 +69,6 @@ from openai.types.chat import (completion_create_params,)from openai.types.chat.chat_completion import Choice-from openai.types.chat.chat_completion_chunk import Choice as ChunkChoicefrom openai.types.shared_params import (FunctionDefinition,FunctionParameters,@@ -724,21 +723,19 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):)# Prepare data to process streaming chunks.- choice: Union[ParsedChoice[Any], ParsedChoice[BaseModel], ChunkChoice] = cast(ChunkChoice, None)- chunk = None+ chunk: ChatCompletionChunk | None = Nonestop_reason = Nonemaybe_model = Nonecontent_deltas: List[str] = []thought_deltas: List[str] = []full_tool_calls: Dict[int, FunctionCall] = {}- completion_tokens = 0logprobs: Optional[List[ChatCompletionTokenLogprob]] = Noneempty_chunk_warning_has_been_issued: bool = Falseempty_chunk_warning_threshold: int = 10empty_chunk_count = 0-first_chunk = True+ is_reasoning = False# Process the stream of chunks.async for chunk in chunks:@@ -750,6 +747,10 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):messages=cast(List[Dict[str, Any]], create_params.messages),))++ # Set the model from the lastest chunk.+ maybe_model = chunk.model+# Empty chunks has been observed when the endpoint is under heavy load.# https://github.com/microsoft/autogen/issues/4213if len(chunk.choices) == 0:@@ -764,21 +765,41 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):else:empty_chunk_count = 0- # to process usage chunk in streaming situations- # add stream_options={"include_usage": True} in the initialization of OpenAIChatCompletionClient(...)- # However the different api's- # OPENAI api usage chunk produces no choices so need to check if there is a choice- # liteLLM api usage chunk does produce choices- choice = (- chunk.choices[0]- if len(chunk.choices) > 0- else (choice if chunk.usage is not None and stop_reason is not None else cast(ChunkChoice, None))- )+ if len(chunk.choices) > 1:+ # This is a multi-choice chunk, we need to warn the user.+ warnings.warn(+ f"Received a chunk with {len(chunk.choices)} choices. Only the first choice will be used.",+ UserWarning,+ stacklevel=2,+ )++ # Set the choice to the first choice in the chunk.+ choice = chunk.choices[0]# for liteLLM chunk usage, do the following hack keeping the pervious chunk.stop_reason (if set).# set the stop_reason for the usage chunk to the prior stop_reasonstop_reason = choice.finish_reason if chunk.usage is None and stop_reason is None else stop_reasonmaybe_model = chunk.model++ reasoning_content: str | None = None+ if choice.delta.model_extra is not None and "reasoning_content" in choice.delta.model_extra:+ # If there is a reasoning_content field, then we populate the thought field. This is for models such as R1.+ reasoning_content = choice.delta.model_extra.get("reasoning_content")++ if isinstance(reasoning_content, str) and len(reasoning_content) > 0:+ if not is_reasoning:+ # Enter reasoning mode.+ reasoning_content = "<think>" + reasoning_content+ is_reasoning = True+ thought_deltas.append(reasoning_content)+ yield reasoning_content+ elif is_reasoning:+ # Exit reasoning mode.+ reasoning_content = "</think>"+ thought_deltas.append(reasoning_content)+ is_reasoning = False+ yield reasoning_content+# First try get contentif choice.delta.content:content_deltas.append(choice.delta.content)@@ -787,12 +808,6 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):# NOTE: for OpenAI, tool_calls and content are mutually exclusive it seems, so we can skip the rest of the loop.# However, this may not be the case for other APIs -- we should expect this may need to be updated.continue- # if there is a reasoning_content field, then we populate the thought field. This is for models such as R1.- if choice.delta.model_extra is not None:- reasoning_content = choice.delta.model_extra.get("reasoning_content")- if reasoning_content is not None:- thought_deltas.append(reasoning_content)- yield reasoning_content# Otherwise, get tool callsif choice.delta.tool_calls is not None:for tool_call_chunk in choice.delta.tool_calls:@@ -863,13 +878,13 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):)content = ""- # Always set thoughts if we have any, regardless of other content types- if thought_deltas:- thought = "".join(thought_deltas)+ # Set thoughts if we have any reasoning content.+ if thought_deltas:+ thought = "".join(thought_deltas).lstrip("<think>").rstrip("</think>")- # This is for local R1 models.- if isinstance(content, str) and self._model_info["family"] == ModelFamily.R1 and thought is None:- thought, content = parse_r1_content(content)+ # This is for local R1 models whose reasoning content is within the content string.+ if isinstance(content, str) and self._model_info["family"] == ModelFamily.R1 and thought is None:+ thought, content = parse_r1_content(content)# Create the result.result = CreateResult(diff --git a/python/packages/autogen-ext/tests/models/test_openai_model_client.py b/python/packages/autogen-ext/tests/models/test_openai_model_client.pyindex 6fcc6dc3b..7ae6aeee0 100644--- a/python/packages/autogen-ext/tests/models/test_openai_model_client.py+++ b/python/packages/autogen-ext/tests/models/test_openai_model_client.py@@ -1090,13 +1090,14 @@ async def test_r1_reasoning_content_streaming(monkeypatch: pytest.MonkeyPatch) -# Verify that the chunks first stream the reasoning content and then the main content# Then verify that the final result has the correct content and thought- assert len(chunks) == 4- assert chunks[0] == "This is the reasoning content 1"+ assert len(chunks) == 5+ assert chunks[0] == "<think>This is the reasoning content 1"assert chunks[1] == "This is the reasoning content 2"- assert chunks[2] == "This is the main content"- assert isinstance(chunks[3], CreateResult)- assert chunks[3].content == "This is the main content"- assert chunks[3].thought == "This is the reasoning content 1This is the reasoning content 2"+ assert chunks[2] == "</think>"+ assert chunks[3] == "This is the main content"+ assert isinstance(chunks[4], CreateResult)+ assert chunks[4].content == "This is the main content"+ assert chunks[4].thought == "This is the reasoning content 1This is the reasoning content 2"@pytest.mark.asyncio@@ -1783,7 +1784,10 @@ async def test_openai_structured_output_with_streaming(model: str, openai_client@pytest.mark.asyncio@pytest.mark.parametrize("model",- ["gpt-4o-mini", "gemini-1.5-flash"],+ [+ "gpt-4o-mini",+ # "gemini-1.5-flash", # Gemini models do not support structured output with tool calls from model client.+ ],)async def test_openai_structured_output_with_tool_calls(model: str, openai_client: OpenAIChatCompletionClient) -> None:class AgentResponse(BaseModel):@@ -1796,13 +1800,15 @@ async def test_openai_structured_output_with_tool_calls(model: str, openai_clientool = FunctionTool(sentiment_analysis, description="Sentiment Analysis", strict=True)+ extra_create_args = {"tool_choice": "required"}+response1 = await openai_client.create(messages=[SystemMessage(content="Analyze input text sentiment using the tool provided."),UserMessage(content="I am happy.", source="user"),],tools=[tool],- extra_create_args={"tool_choice": "required"},+ extra_create_args=extra_create_args,json_output=AgentResponse,)assert isinstance(response1.content, list)@@ -1836,7 +1842,10 @@ async def test_openai_structured_output_with_tool_calls(model: str, openai_clien@pytest.mark.asyncio@pytest.mark.parametrize("model",- ["gpt-4o-mini", "gemini-1.5-flash"],+ [+ "gpt-4o-mini",+ # "gemini-1.5-flash", # Gemini models do not support structured output with tool calls from model client.+ ],)async def test_openai_structured_output_with_streaming_tool_calls(model: str, openai_client: OpenAIChatCompletionClient@@ -1851,6 +1860,8 @@ async def test_openai_structured_output_with_streaming_tool_calls(tool = FunctionTool(sentiment_analysis, description="Sentiment Analysis", strict=True)+ extra_create_args = {"tool_choice": "required"}+chunks1: List[str | CreateResult] = []stream1 = openai_client.create_stream(messages=[@@ -1858,7 +1869,7 @@ async def test_openai_structured_output_with_streaming_tool_calls(UserMessage(content="I am happy.", source="user"),],tools=[tool],- extra_create_args={"tool_choice": "required"},+ extra_create_args=extra_create_args,json_output=AgentResponse,)async for chunk in stream1: