A programming framework for agentic AI
Add reply chat completion client (#4096)
Mohammad Mazraeh570471bed7dd
diff --git a/python/packages/autogen-ext/src/autogen_ext/models/__init__.py b/python/packages/autogen-ext/src/autogen_ext/models/__init__.pyindex e7b2b76ae..d39c1d9bf 100644--- a/python/packages/autogen-ext/src/autogen_ext/models/__init__.py+++ b/python/packages/autogen-ext/src/autogen_ext/models/__init__.py@@ -2,8 +2,6 @@ from ._openai._openai_client import (AzureOpenAIChatCompletionClient,OpenAIChatCompletionClient,)+from ._reply_chat_completion_client import ReplayChatCompletionClient-__all__ = [- "AzureOpenAIChatCompletionClient",- "OpenAIChatCompletionClient",-]+__all__ = ["AzureOpenAIChatCompletionClient", "OpenAIChatCompletionClient", "ReplayChatCompletionClient"]diff --git a/python/packages/autogen-ext/src/autogen_ext/models/_reply_chat_completion_client.py b/python/packages/autogen-ext/src/autogen_ext/models/_reply_chat_completion_client.pynew file mode 100644index 000000000..a4889f44b--- /dev/null+++ b/python/packages/autogen-ext/src/autogen_ext/models/_reply_chat_completion_client.py@@ -0,0 +1,218 @@+from __future__ import annotations++import logging+from typing import Any, AsyncGenerator, List, Mapping, Optional, Sequence, Union++from autogen_core.application.logging import EVENT_LOGGER_NAME+from autogen_core.base import CancellationToken+from autogen_core.components.models import (+ ChatCompletionClient,+ CreateResult,+ LLMMessage,+ ModelCapabilities,+ RequestUsage,+)+from autogen_core.components.tools import Tool, ToolSchema++logger = logging.getLogger(EVENT_LOGGER_NAME)+++class ReplayChatCompletionClient:+ """+ A mock chat completion client that replays predefined responses using an index-based approach.++ This class simulates a chat completion client by replaying a predefined list of responses. It supports both single completion and streaming responses. The responses can be either strings or CreateResult objects. The client now uses an index-based approach to access the responses, allowing for resetting the state.++ .. note::+ The responses can be either strings or CreateResult objects.++ Args:+ chat_completions (Sequence[Union[str, CreateResult]]): A list of predefined responses to replay.++ Raises:+ ValueError("No more mock responses available"): If the list of provided outputs are exhausted.++ Examples:++ Simple chat completion client to return pre-defined responses.++ .. code-block:: python++ chat_completions = [+ "Hello, how can I assist you today?",+ "I'm happy to help with any questions you have.",+ "Is there anything else I can assist you with?",+ ]+ client = ReplayChatCompletionClient(chat_completions)+ messages = [LLMMessage(content="What can you do?")]+ response = await client.create(messages)+ print(response.content) # Output: "Hello, how can I assist you today?"++ Simple streaming chat completion client to return pre-defined responses++ .. code-block:: python++ chat_completions = [+ "Hello, how can I assist you today?",+ "I'm happy to help with any questions you have.",+ "Is there anything else I can assist you with?",+ ]+ client = ReplayChatCompletionClient(chat_completions)++ async for token in client.create_stream(messages):+ print(token, end="") # Output: "Hello, how can I assist you today?"++ async for token in client.create_stream(messages):+ print(token, end="") # Output: "I'm happy to help with any questions you have."++ Using `.reset` to reset the chat client state++ .. code-block:: python++ chat_completions = [+ "Hello, how can I assist you today?",+ ]+ client = ReplayChatCompletionClient(chat_completions)+ messages = [LLMMessage(content="What can you do?")]+ response = await client.create(messages)+ print(response.content) # Output: "Hello, how can I assist you today?"++ response = await client.create(messages) # Raises ValueError("No more mock responses available")++ client.reset() # Reset the client state (current index of message and token usages)+ response = await client.create(messages)+ print(response.content) # Output: "Hello, how can I assist you today?" again++ """++ __protocol__: ChatCompletionClient++ # TODO: Support FunctionCall in responses+ # TODO: Support logprobs in Responses+ # TODO: Support model capabilities++ def __init__(+ self,+ chat_completions: Sequence[Union[str, CreateResult]],+ ):+ self.chat_completions = list(chat_completions)+ self.provided_message_count = len(self.chat_completions)+ self._model_capabilities = ModelCapabilities(vision=False, function_calling=False, json_output=False)+ self._total_available_tokens = 10000+ self._cur_usage = RequestUsage(prompt_tokens=0, completion_tokens=0)+ self._total_usage = RequestUsage(prompt_tokens=0, completion_tokens=0)+ self._current_index = 0++ async def create(+ self,+ messages: Sequence[LLMMessage],+ tools: Sequence[Tool | ToolSchema] = [],+ json_output: Optional[bool] = None,+ extra_create_args: Mapping[str, Any] = {},+ cancellation_token: Optional[CancellationToken] = None,+ ) -> CreateResult:+ """Return the next completion from the list."""+ if self._current_index >= len(self.chat_completions):+ raise ValueError("No more mock responses available")++ response = self.chat_completions[self._current_index]+ _, prompt_token_count = self._tokenize(messages)+ if isinstance(response, str):+ _, output_token_count = self._tokenize(response)+ self._cur_usage = RequestUsage(prompt_tokens=prompt_token_count, completion_tokens=output_token_count)+ response = CreateResult(finish_reason="stop", content=response, usage=self._cur_usage, cached=True)+ else:+ self._cur_usage = RequestUsage(+ prompt_tokens=prompt_token_count, completion_tokens=response.usage.completion_tokens+ )++ self._update_total_usage()+ self._current_index += 1+ return response++ async def create_stream(+ self,+ messages: Sequence[LLMMessage],+ tools: Sequence[Tool | ToolSchema] = [],+ json_output: Optional[bool] = None,+ extra_create_args: Mapping[str, Any] = {},+ cancellation_token: Optional[CancellationToken] = None,+ ) -> AsyncGenerator[Union[str, CreateResult], None]:+ """Return the next completion as a stream."""+ if self._current_index >= len(self.chat_completions):+ raise ValueError("No more mock responses available")++ response = self.chat_completions[self._current_index]+ _, prompt_token_count = self._tokenize(messages)+ if isinstance(response, str):+ output_tokens, output_token_count = self._tokenize(response)+ self._cur_usage = RequestUsage(prompt_tokens=prompt_token_count, completion_tokens=output_token_count)++ for i, token in enumerate(output_tokens):+ if i < len(output_tokens) - 1:+ yield token + " "+ else:+ yield token+ self._update_total_usage()+ else:+ self._cur_usage = RequestUsage(+ prompt_tokens=prompt_token_count, completion_tokens=response.usage.completion_tokens+ )+ yield response+ self._update_total_usage()++ self._current_index += 1++ def actual_usage(self) -> RequestUsage:+ return self._cur_usage++ def total_usage(self) -> RequestUsage:+ return self._total_usage++ def count_tokens(self, messages: Sequence[LLMMessage], tools: Sequence[Tool | ToolSchema] = []) -> int:+ _, token_count = self._tokenize(messages)+ return token_count++ def remaining_tokens(self, messages: Sequence[LLMMessage], tools: Sequence[Tool | ToolSchema] = []) -> int:+ return max(+ 0, self._total_available_tokens - self._total_usage.prompt_tokens - self._total_usage.completion_tokens+ )++ def _tokenize(self, messages: Union[str, LLMMessage, Sequence[LLMMessage]]) -> tuple[list[str], int]:+ total_tokens = 0+ all_tokens: List[str] = []+ if isinstance(messages, str):+ tokens = messages.split()+ total_tokens += len(tokens)+ all_tokens.extend(tokens)+ elif hasattr(messages, "content"):+ if isinstance(messages.content, str): # type: ignore [reportAttributeAccessIssue]+ tokens = messages.content.split() # type: ignore [reportAttributeAccessIssue]+ total_tokens += len(tokens)+ all_tokens.extend(tokens)+ else:+ logger.warning("Token count has been done only on string content", RuntimeWarning)+ elif isinstance(messages, Sequence):+ for message in messages:+ if isinstance(message.content, str): # type: ignore [reportAttributeAccessIssue, union-attr]+ tokens = message.content.split() # type: ignore [reportAttributeAccessIssue, union-attr]+ total_tokens += len(tokens)+ all_tokens.extend(tokens)+ else:+ logger.warning("Token count has been done only on string content", RuntimeWarning)+ return all_tokens, total_tokens++ def _update_total_usage(self) -> None:+ self._total_usage.completion_tokens += self._cur_usage.completion_tokens+ self._total_usage.prompt_tokens += self._cur_usage.prompt_tokens++ @property+ def capabilities(self) -> ModelCapabilities:+ """Return mock capabilities."""+ return self._model_capabilities++ def reset(self) -> None:+ """Reset the client state and usage to its initial state."""+ self._cur_usage = RequestUsage(prompt_tokens=0, completion_tokens=0)+ self._total_usage = RequestUsage(prompt_tokens=0, completion_tokens=0)+ self._current_index = 0diff --git a/python/packages/autogen-ext/tests/models/test_reply_chat_completion_client.py b/python/packages/autogen-ext/tests/models/test_reply_chat_completion_client.pynew file mode 100644index 000000000..2f0ba52a8--- /dev/null+++ b/python/packages/autogen-ext/tests/models/test_reply_chat_completion_client.py@@ -0,0 +1,172 @@+import copy+from dataclasses import dataclass+from typing import List++import pytest+from autogen_core.application import SingleThreadedAgentRuntime+from autogen_core.base import AgentId, MessageContext+from autogen_core.components import DefaultTopicId, RoutedAgent, default_subscription, message_handler+from autogen_core.components.models import ChatCompletionClient, CreateResult, SystemMessage, UserMessage+from autogen_ext.models import ReplayChatCompletionClient+++@dataclass+class ContentMessage:+ content: str+++class LLMAgent(RoutedAgent):+ def __init__(self, model_client: ChatCompletionClient) -> None:+ super().__init__("LLM Agent!")+ self._chat_history: List[ContentMessage] = []+ self._model_client = model_client+ self.num_calls = 0++ @message_handler+ async def on_new_message(self, message: ContentMessage, ctx: MessageContext) -> None:+ self._chat_history.append(message)+ self.num_calls += 1+ completion = await self._model_client.create(messages=self._fixed_message_history_type)+ if isinstance(completion.content, str):+ await self.publish_message(ContentMessage(content=completion.content), DefaultTopicId())+ else:+ raise TypeError(f"Completion content of type {type(completion.content)} is not supported")++ @property+ def _fixed_message_history_type(self) -> List[SystemMessage]:+ return [SystemMessage(msg.content) for msg in self._chat_history]+++@default_subscription+class LLMAgentWithDefaultSubscription(LLMAgent): ...+++async def test_reply_chat_completion_client() -> None:+ num_messages = 5+ messages = [f"Message {i}" for i in range(num_messages)]+ reply_model_client = ReplayChatCompletionClient(messages)++ for i in range(num_messages):+ completion: CreateResult = await reply_model_client.create([UserMessage(content="dummy", source="_")])+ assert completion.content == messages[i]+ with pytest.raises(ValueError, match="No more mock responses available"):+ await reply_model_client.create([UserMessage(content="dummy", source="_")])+++async def test_reply_chat_completion_client_create_stream() -> None:+ num_messages = 5+ messages = [f"Message {i}" for i in range(num_messages)]+ reply_model_client = ReplayChatCompletionClient(messages)++ for i in range(num_messages):+ result: List[str] = []+ async for completion in reply_model_client.create_stream([UserMessage(content="dummy", source="_")]):+ text = completion.content if isinstance(completion, CreateResult) else completion+ assert isinstance(text, str)+ result.append(text)+ assert "".join(result) == messages[i]++ with pytest.raises(ValueError, match="No more mock responses available"):+ await reply_model_client.create([UserMessage(content="dummy", source="_")])+++async def test_register_receives_publish_llm() -> None:+ runtime = SingleThreadedAgentRuntime()+ runtime.start()++ reply_model_client_1 = ReplayChatCompletionClient(["Hi!", "Doing Good, you?", "Bye!"])+ reply_model_client_2 = ReplayChatCompletionClient(["Hi! How are you doing?", "Good, nice to meet you, bye!"])++ # First registered models gets the first message+ assert reply_model_client_1.provided_message_count == 1 + reply_model_client_2.provided_message_count++ await LLMAgentWithDefaultSubscription.register(+ runtime, "LLMAgent1", lambda: LLMAgentWithDefaultSubscription(reply_model_client_1)+ )++ await LLMAgentWithDefaultSubscription.register(+ runtime, "LLMAgent2", lambda: LLMAgentWithDefaultSubscription(reply_model_client_2)+ )++ await runtime.publish_message(ContentMessage(content="Let's get started!"), DefaultTopicId())+ await runtime.stop_when_idle()++ agent_1 = await runtime.try_get_underlying_agent_instance(+ AgentId("LLMAgent1", key="default"), type=LLMAgentWithDefaultSubscription+ )++ agent_2 = await runtime.try_get_underlying_agent_instance(+ AgentId("LLMAgent2", key="default"), type=LLMAgentWithDefaultSubscription+ )++ assert agent_1.num_calls == 1 + reply_model_client_2.provided_message_count+ assert agent_2.num_calls == 1 + reply_model_client_1.provided_message_count+++async def test_token_count_logics() -> None:+ phrases = [+ "This is a test message.",+ "This is another test message.",+ "This is yet another test message.",+ "Maybe even more messages?",+ ]+ reply_model_client = ReplayChatCompletionClient(phrases)++ messages = [UserMessage(content="How many tokens are in this message?", source="_")]++ token_count = reply_model_client.count_tokens(messages)+ assert token_count == 7++ _ = await reply_model_client.create(messages)+ remaining_tokens = reply_model_client.remaining_tokens(messages)+ assert remaining_tokens == 9988++ multiple_messages = [UserMessage(content="This is another test message.", source="_")]+ total_token_count = reply_model_client.count_tokens(messages + multiple_messages)+ assert total_token_count == 12++ before_cteate_usage = copy.deepcopy(reply_model_client.total_usage())+ completion: CreateResult = await reply_model_client.create(messages)++ assert completion.usage.prompt_tokens == 7+ assert completion.usage.completion_tokens == 5++ after_create_usage = reply_model_client.total_usage()+ assert after_create_usage.prompt_tokens > before_cteate_usage.prompt_tokens+ assert after_create_usage.completion_tokens > before_cteate_usage.completion_tokens++ before_cteate_stream_usage = copy.deepcopy(reply_model_client.total_usage())++ async for _ in reply_model_client.create_stream(messages):+ pass+ after_create_stream_usage = reply_model_client.total_usage()+ assert after_create_stream_usage.completion_tokens > before_cteate_stream_usage.completion_tokens+ assert after_create_stream_usage.prompt_tokens > before_cteate_stream_usage.prompt_tokens+++async def test_reply_chat_completion_client_reset() -> None:+ """Test that reset functionality properly resets the client state."""+ messages = ["First message", "Second message", "Third message"]+ client = ReplayChatCompletionClient(messages)++ # Use all messages once+ for expected_msg in messages:+ completion = await client.create([UserMessage(content="dummy", source="_")])+ assert completion.content == expected_msg++ # Should raise error when no more messages+ with pytest.raises(ValueError, match="No more mock responses available"):+ await client.create([UserMessage(content="dummy", source="_")])++ # Reset the client+ client.reset()++ # Should be able to get all messages again in the same order+ for expected_msg in messages:+ completion = await client.create([UserMessage(content="dummy", source="_")])+ assert completion.content == expected_msg