Skip to content

A programming framework for agentic AI

add tolerance for empty choices (#4311)

Mohammad Mazraeh341417e6367f
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/_openai/_openai_client.py b/python/packages/autogen-ext/src/autogen_ext/models/_openai/_openai_client.py
index 7faefa44c..999e5083f 100644
--- a/python/packages/autogen-ext/src/autogen_ext/models/_openai/_openai_client.py
+++ b/python/packages/autogen-ext/src/autogen_ext/models/_openai/_openai_client.py
@@ -556,6 +556,8 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
json_output: Optional[bool] = None,
extra_create_args: Mapping[str, Any] = {},
cancellation_token: Optional[CancellationToken] = None,
+ *,
+ max_consecutive_empty_chunk_tolerance: int = 0,
) -> AsyncGenerator[Union[str, CreateResult], None]:
"""
Creates an AsyncGenerator that will yield a stream of chat completions based on the provided messages and tools.
@@ -566,6 +568,7 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
json_output (Optional[bool], optional): If True, the output will be in JSON format. Defaults to None.
extra_create_args (Mapping[str, Any], optional): Additional arguments for the creation process. Default to `{}`.
cancellation_token (Optional[CancellationToken], optional): A token to cancel the operation. Defaults to None.
+ max_consecutive_empty_chunk_tolerance (int): The maximum number of consecutive empty chunks to tolerate before raising a ValueError. This seems to only be needed to set when using `AzureOpenAIChatCompletionClient`. Defaults to 0.
Yields:
AsyncGenerator[Union[str, CreateResult], None]: A generator yielding the completion results as they are produced.
@@ -636,6 +639,8 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
full_tool_calls: Dict[int, FunctionCall] = {}
completion_tokens = 0
logprobs: Optional[List[ChatCompletionTokenLogprob]] = None
+ empty_chunk_count = 0
+
while True:
try:
chunk_future = asyncio.ensure_future(anext(stream))
@@ -643,6 +648,20 @@ class BaseOpenAIChatCompletionClient(ChatCompletionClient):
cancellation_token.link_future(chunk_future)
chunk = await chunk_future
+ # This is to address a bug in AzureOpenAIChatCompletionClient. OpenAIChatCompletionClient works fine.
+ # https://github.com/microsoft/autogen/issues/4213
+ if len(chunk.choices) == 0:
+ empty_chunk_count += 1
+ if max_consecutive_empty_chunk_tolerance == 0:
+ raise ValueError(
+ "Consecutive empty chunks found. Change max_empty_consecutive_chunk_tolerance to increase empty chunk tolerance"
+ )
+ elif empty_chunk_count >= max_consecutive_empty_chunk_tolerance:
+ raise ValueError("Exceeded the threshold of receiving consecutive empty chunks")
+ continue
+ else:
+ empty_chunk_count = 0
+
# to process usage chunk in streaming situations
# add stream_options={"include_usage": True} in the initialization of OpenAIChatCompletionClient(...)
# However the different api's