The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
openai[patch]: support streaming with json_schema response format (#29044)
ccurme815bfa1913d0
diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.pyindex 142e7eca1a..f4e2625348 100644--- a/libs/partners/openai/langchain_openai/chat_models/base.py+++ b/libs/partners/openai/langchain_openai/chat_models/base.py@@ -318,8 +318,14 @@ def _convert_delta_to_message_chunk(def _convert_chunk_to_generation_chunk(chunk: dict, default_chunk_class: Type, base_generation_info: Optional[Dict]) -> Optional[ChatGenerationChunk]:+ if chunk.get("type") == "content.delta": # from beta.chat.completions.stream+ return Nonetoken_usage = chunk.get("usage")- choices = chunk.get("choices", [])+ choices = (+ chunk.get("choices", [])+ # from beta.chat.completions.stream+ or chunk.get("chunk", {}).get("choices", [])+ )usage_metadata: Optional[UsageMetadata] = (_create_usage_metadata(token_usage) if token_usage else None@@ -660,13 +666,24 @@ class BaseChatOpenAI(BaseChatModel):default_chunk_class: Type[BaseMessageChunk] = AIMessageChunkbase_generation_info = {}- if self.include_response_headers:- raw_response = self.client.with_raw_response.create(**payload)- response = raw_response.parse()- base_generation_info = {"headers": dict(raw_response.headers)}+ if "response_format" in payload:+ if self.include_response_headers:+ warnings.warn(+ "Cannot currently include response headers when response_format is "+ "specified."+ )+ payload.pop("stream")+ response_stream = self.root_client.beta.chat.completions.stream(**payload)+ context_manager = response_streamelse:- response = self.client.create(**payload)- with response:+ if self.include_response_headers:+ raw_response = self.client.with_raw_response.create(**payload)+ response = raw_response.parse()+ base_generation_info = {"headers": dict(raw_response.headers)}+ else:+ response = self.client.create(**payload)+ context_manager = response+ with context_manager as response:is_first_chunk = Truefor chunk in response:if not isinstance(chunk, dict):@@ -686,6 +703,16 @@ class BaseChatOpenAI(BaseChatModel):)is_first_chunk = Falseyield generation_chunk+ if hasattr(response, "get_final_completion") and "response_format" in payload:+ final_completion = response.get_final_completion()+ generation_chunk = self._get_generation_chunk_from_completion(+ final_completion+ )+ if run_manager:+ run_manager.on_llm_new_token(+ generation_chunk.text, chunk=generation_chunk+ )+ yield generation_chunkdef _generate(self,@@ -794,13 +821,29 @@ class BaseChatOpenAI(BaseChatModel):payload = self._get_request_payload(messages, stop=stop, **kwargs)default_chunk_class: Type[BaseMessageChunk] = AIMessageChunkbase_generation_info = {}- if self.include_response_headers:- raw_response = await self.async_client.with_raw_response.create(**payload)- response = raw_response.parse()- base_generation_info = {"headers": dict(raw_response.headers)}++ if "response_format" in payload:+ if self.include_response_headers:+ warnings.warn(+ "Cannot currently include response headers when response_format is "+ "specified."+ )+ payload.pop("stream")+ response_stream = self.root_async_client.beta.chat.completions.stream(+ **payload+ )+ context_manager = response_streamelse:- response = await self.async_client.create(**payload)- async with response:+ if self.include_response_headers:+ raw_response = await self.async_client.with_raw_response.create(+ **payload+ )+ response = raw_response.parse()+ base_generation_info = {"headers": dict(raw_response.headers)}+ else:+ response = await self.async_client.create(**payload)+ context_manager = response+ async with context_manager as response:is_first_chunk = Trueasync for chunk in response:if not isinstance(chunk, dict):@@ -820,6 +863,16 @@ class BaseChatOpenAI(BaseChatModel):)is_first_chunk = Falseyield generation_chunk+ if hasattr(response, "get_final_completion") and "response_format" in payload:+ final_completion = await response.get_final_completion()+ generation_chunk = self._get_generation_chunk_from_completion(+ final_completion+ )+ if run_manager:+ await run_manager.on_llm_new_token(+ generation_chunk.text, chunk=generation_chunk+ )+ yield generation_chunkasync def _agenerate(self,@@ -1010,25 +1063,6 @@ class BaseChatOpenAI(BaseChatModel):num_tokens += 3return num_tokens- def _should_stream(- self,- *,- async_api: bool,- run_manager: Optional[- Union[CallbackManagerForLLMRun, AsyncCallbackManagerForLLMRun]- ] = None,- response_format: Optional[Union[dict, type]] = None,- **kwargs: Any,- ) -> bool:- if isinstance(response_format, type) and is_basemodel_subclass(response_format):- # TODO: Add support for streaming with Pydantic response_format.- warnings.warn("Streaming with Pydantic response_format not yet supported.")- return False-- return super()._should_stream(- async_api=async_api, run_manager=run_manager, **kwargs- )-@deprecated(since="0.2.1",alternative="langchain_openai.chat_models.base.ChatOpenAI.bind_tools",@@ -1531,6 +1565,25 @@ class BaseChatOpenAI(BaseChatModel):filtered[k] = vreturn filtered+ def _get_generation_chunk_from_completion(+ self, completion: openai.BaseModel+ ) -> ChatGenerationChunk:+ """Get chunk from completion (e.g., from final completion of a stream)."""+ chat_result = self._create_chat_result(completion)+ chat_message = chat_result.generations[0].message+ if isinstance(chat_message, AIMessage):+ usage_metadata = chat_message.usage_metadata+ else:+ usage_metadata = None+ message = AIMessageChunk(+ content="",+ additional_kwargs=chat_message.additional_kwargs,+ usage_metadata=usage_metadata,+ )+ return ChatGenerationChunk(+ message=message, generation_info=chat_result.llm_output+ )+class ChatOpenAI(BaseChatOpenAI): # type: ignore[override]"""OpenAI chat model integration.diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_azure.py b/libs/partners/openai/tests/integration_tests/chat_models/test_azure.pyindex 4ed531ad11..e9228df073 100644--- a/libs/partners/openai/tests/integration_tests/chat_models/test_azure.py+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_azure.py@@ -13,6 +13,7 @@ from langchain_core.messages import (HumanMessage,)from langchain_core.outputs import ChatGeneration, ChatResult, LLMResult+from pydantic import BaseModelfrom langchain_openai import AzureChatOpenAIfrom tests.unit_tests.fake.callbacks import FakeCallbackHandler@@ -262,3 +263,37 @@ async def test_json_mode_async(llm: AzureChatOpenAI) -> None:assert isinstance(full, AIMessageChunk)assert isinstance(full.content, str)assert json.loads(full.content) == {"a": 1}+++class Foo(BaseModel):+ response: str+++def test_stream_response_format(llm: AzureChatOpenAI) -> None:+ full: Optional[BaseMessageChunk] = None+ chunks = []+ for chunk in llm.stream("how are ya", response_format=Foo):+ chunks.append(chunk)+ full = chunk if full is None else full + chunk+ assert len(chunks) > 1+ assert isinstance(full, AIMessageChunk)+ parsed = full.additional_kwargs["parsed"]+ assert isinstance(parsed, Foo)+ assert isinstance(full.content, str)+ parsed_content = json.loads(full.content)+ assert parsed.response == parsed_content["response"]+++async def test_astream_response_format(llm: AzureChatOpenAI) -> None:+ full: Optional[BaseMessageChunk] = None+ chunks = []+ async for chunk in llm.astream("how are ya", response_format=Foo):+ chunks.append(chunk)+ full = chunk if full is None else full + chunk+ assert len(chunks) > 1+ assert isinstance(full, AIMessageChunk)+ parsed = full.additional_kwargs["parsed"]+ assert isinstance(parsed, Foo)+ assert isinstance(full.content, str)+ parsed_content = json.loads(full.content)+ assert parsed.response == parsed_content["response"]diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base.pyindex 93c08ce214..506799aef4 100644--- a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py+++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py@@ -1092,14 +1092,37 @@ class Foo(BaseModel):def test_stream_response_format() -> None:- list(ChatOpenAI(model="gpt-4o-mini").stream("how are ya", response_format=Foo))+ full: Optional[BaseMessageChunk] = None+ chunks = []+ for chunk in ChatOpenAI(model="gpt-4o-mini").stream(+ "how are ya", response_format=Foo+ ):+ chunks.append(chunk)+ full = chunk if full is None else full + chunk+ assert len(chunks) > 1+ assert isinstance(full, AIMessageChunk)+ parsed = full.additional_kwargs["parsed"]+ assert isinstance(parsed, Foo)+ assert isinstance(full.content, str)+ parsed_content = json.loads(full.content)+ assert parsed.response == parsed_content["response"]async def test_astream_response_format() -> None:- async for _ in ChatOpenAI(model="gpt-4o-mini").astream(+ full: Optional[BaseMessageChunk] = None+ chunks = []+ async for chunk in ChatOpenAI(model="gpt-4o-mini").astream("how are ya", response_format=Foo):- pass+ chunks.append(chunk)+ full = chunk if full is None else full + chunk+ assert len(chunks) > 1+ assert isinstance(full, AIMessageChunk)+ parsed = full.additional_kwargs["parsed"]+ assert isinstance(parsed, Foo)+ assert isinstance(full.content, str)+ parsed_content = json.loads(full.content)+ assert parsed.response == parsed_content["response"]@pytest.mark.parametrize("use_max_completion_tokens", [True, False])