A programming framework for agentic AI
Made the cost info easier to read (#2356)
HRUSHIKESH DOKALA59daf78d9fb6
diff --git a/autogen/agentchat/chat.py b/autogen/agentchat/chat.pyindex a07f3302a..10ad8014e 100644--- a/autogen/agentchat/chat.py+++ b/autogen/agentchat/chat.py@@ -25,10 +25,12 @@ class ChatResult:"""The chat history."""summary: str = None"""A summary obtained from the chat."""- cost: tuple = None # (dict, dict) - (total_cost, actual_cost_with_cache)- """The cost of the chat. a tuple of (total_cost, total_actual_cost), where total_cost is a- dictionary of cost information, and total_actual_cost is a dictionary of information on- the actual incurred cost with cache."""+ cost: Dict[str, dict] = None # keys: "usage_including_cached_inference", "usage_excluding_cached_inference"+ """The cost of the chat.+ The value for each usage type is a dictionary containing cost information for that specific type.+ - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference.+ - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference".+ """human_input: List[str] = None"""A list of human input solicited during the chat."""diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.pyindex eef374160..b32c2f5f0 100644--- a/autogen/agentchat/utils.py+++ b/autogen/agentchat/utils.py@@ -1,5 +1,5 @@import re-from typing import Any, Callable, Dict, List, Tuple, Union+from typing import Any, Callable, Dict, List, Unionfrom .agent import Agent@@ -26,33 +26,46 @@ def consolidate_chat_info(chat_info, uniform_sender=None) -> None:), "llm client must be set in either the recipient or sender when summary_method is reflection_with_llm."-def gather_usage_summary(agents: List[Agent]) -> Tuple[Dict[str, any], Dict[str, any]]:+def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, Dict], Dict[str, Dict]]:r"""Gather usage summary from all agents.Args:agents: (list): List of agents.Returns:- tuple: (total_usage_summary, actual_usage_summary)+ dictionary: A dictionary containing two keys:+ - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference.+ - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference".Example:```python- total_usage_summary = {- "total_cost": 0.0006090000000000001,- "gpt-35-turbo": {- "cost": 0.0006090000000000001,- "prompt_tokens": 242,- "completion_tokens": 123,- "total_tokens": 365+ {+ "usage_including_cached_inference" : {+ "total_cost": 0.0006090000000000001,+ "gpt-35-turbo": {+ "cost": 0.0006090000000000001,+ "prompt_tokens": 242,+ "completion_tokens": 123,+ "total_tokens": 365+ },+ },++ "usage_excluding_cached_inference" : {+ "total_cost": 0.0006090000000000001,+ "gpt-35-turbo": {+ "cost": 0.0006090000000000001,+ "prompt_tokens": 242,+ "completion_tokens": 123,+ "total_tokens": 365+ },}}```Note:- `actual_usage_summary` follows the same format.- If none of the agents incurred any cost (not having a client), then the total_usage_summary and actual_usage_summary will be `{'total_cost': 0}`.+ If none of the agents incurred any cost (not having a client), then the usage_including_cached_inference and usage_excluding_cached_inference will be `{'total_cost': 0}`."""def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, Any]) -> None:@@ -69,15 +82,18 @@ def gather_usage_summary(agents: List[Agent]) -> Tuple[Dict[str, any], Dict[str,usage_summary[model]["completion_tokens"] += data.get("completion_tokens", 0)usage_summary[model]["total_tokens"] += data.get("total_tokens", 0)- total_usage_summary = {"total_cost": 0}- actual_usage_summary = {"total_cost": 0}+ usage_including_cached_inference = {"total_cost": 0}+ usage_excluding_cached_inference = {"total_cost": 0}for agent in agents:if getattr(agent, "client", None):- aggregate_summary(total_usage_summary, agent.client.total_usage_summary)- aggregate_summary(actual_usage_summary, agent.client.actual_usage_summary)+ aggregate_summary(usage_including_cached_inference, agent.client.total_usage_summary)+ aggregate_summary(usage_excluding_cached_inference, agent.client.actual_usage_summary)- return total_usage_summary, actual_usage_summary+ return {+ "usage_including_cached_inference": usage_including_cached_inference,+ "usage_excluding_cached_inference": usage_excluding_cached_inference,+ }def parse_tags_from_content(tag: str, content: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Dict[str, str]]]:diff --git a/autogen/code_utils.py b/autogen/code_utils.pyindex aa75756e0..5057c8615 100644--- a/autogen/code_utils.py+++ b/autogen/code_utils.py@@ -41,7 +41,7 @@ logger = logging.getLogger(__name__)def content_str(content: Union[str, List[Union[UserMessageTextContentPart, UserMessageImageContentPart]], None]) -> str:- """Converts the `content` field of an OpenAI merssage into a string format.+ """Converts the `content` field of an OpenAI message into a string format.This function processes content that may be a string, a list of mixed text and image URLs, or None,and converts it into a string. Text is directly appended to the result string, while image URLs arediff --git a/notebook/agentchat_agentoptimizer.ipynb b/notebook/agentchat_agentoptimizer.ipynbindex dd5624458..924f1a557 100644--- a/notebook/agentchat_agentoptimizer.ipynb+++ b/notebook/agentchat_agentoptimizer.ipynb@@ -459,7 +459,7 @@"name": "python","nbconvert_exporter": "python","pygments_lexer": "ipython3",- "version": "3.9.18"+ "version": "3.9.13"}},"nbformat": 4,diff --git a/notebook/agentchat_auto_feedback_from_code_execution.ipynb b/notebook/agentchat_auto_feedback_from_code_execution.ipynbindex 834e1e7df..37584acdc 100644--- a/notebook/agentchat_auto_feedback_from_code_execution.ipynb+++ b/notebook/agentchat_auto_feedback_from_code_execution.ipynb@@ -692,6 +692,7 @@" file_content = \"No data found.\"\n"," return \"Analyze the data and write a brief but engaging blog post. \\n Data: \\n\" + file_content\n","\n",+ "\n","# followup of the previous question\n","chat_res = user_proxy.initiate_chat(\n"," recipient=assistant,\n",diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynbindex 7334f2e36..457d58517 100644--- a/notebook/agentchat_cost_token_tracking.ipynb+++ b/notebook/agentchat_cost_token_tracking.ipynb@@ -494,8 +494,8 @@}],"source": [- "total_usage_summary, actual_usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n",- "total_usage_summary"+ "usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n",+ "usage_summary[\"usage_including_cached_inference\"]"]}],diff --git a/notebook/agentchat_groupchat_stateflow.ipynb b/notebook/agentchat_groupchat_stateflow.ipynbindex 6205e1147..c824046f7 100644--- a/notebook/agentchat_groupchat_stateflow.ipynb+++ b/notebook/agentchat_groupchat_stateflow.ipynb@@ -112,7 +112,6 @@")\n","\n","\n",- "\n","coder = autogen.AssistantAgent(\n"," name=\"Retrieve_Action_1\",\n"," llm_config=gpt4_config,\n",diff --git a/notebook/agentchat_image_generation_capability.ipynb b/notebook/agentchat_image_generation_capability.ipynbindex 7c0c366a5..b5d298d7f 100644--- a/notebook/agentchat_image_generation_capability.ipynb+++ b/notebook/agentchat_image_generation_capability.ipynb@@ -135,6 +135,7 @@" return content[\"text\"].rstrip().endswith(\"TERMINATE\")\n"," return False\n","\n",+ "\n","def critic_agent() -> autogen.ConversableAgent:\n"," return autogen.ConversableAgent(\n"," name=\"critic\",\n",diff --git a/test/agentchat/test_agent_usage.py b/test/agentchat/test_agent_usage.pyindex 212c5513e..e31672cdd 100755--- a/test/agentchat/test_agent_usage.py+++ b/test/agentchat/test_agent_usage.py@@ -1,21 +1,18 @@#!/usr/bin/env python3 -m pytestimport io+import os+import sysfrom contextlib import redirect_stdoutimport pytest-from conftest import skip_openaifrom test_assistant_agent import KEY_LOC, OAI_CONFIG_LISTimport autogenfrom autogen import AssistantAgent, UserProxyAgent, gather_usage_summary-try:- import openai-except ImportError:- skip = True-else:- skip = False or skip_openai+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))+from conftest import skip_openai as skip # noqa: E402@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")@@ -62,11 +59,11 @@ def test_gathering():"gpt-4": {"cost": 0.3, "prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300},}- total_usage, _ = gather_usage_summary([assistant1, assistant2, assistant3])+ total_usage = gather_usage_summary([assistant1, assistant2, assistant3])- assert round(total_usage["total_cost"], 8) == 0.6- assert round(total_usage["gpt-35-turbo"]["cost"], 8) == 0.3- assert round(total_usage["gpt-4"]["cost"], 8) == 0.3+ assert round(total_usage["usage_including_cached_inference"]["total_cost"], 8) == 0.6+ assert round(total_usage["usage_including_cached_inference"]["gpt-35-turbo"]["cost"], 8) == 0.3+ assert round(total_usage["usage_including_cached_inference"]["gpt-4"]["cost"], 8) == 0.3# test when agent doesn't have clientuser_proxy = UserProxyAgent(@@ -77,7 +74,10 @@ def test_gathering():default_auto_reply="That's all. Thank you.",)- total_usage, acutal_usage = gather_usage_summary([user_proxy])+ total_usage = gather_usage_summary([user_proxy])+ total_usage_summary = total_usage["usage_including_cached_inference"]++ print("Total usage summary:", total_usage_summary)@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")