Skip to content

A programming framework for agentic AI

Made the cost info easier to read (#2356)

HRUSHIKESH DOKALA59daf78d9fb6
diff --git a/autogen/agentchat/chat.py b/autogen/agentchat/chat.py
index a07f3302a..10ad8014e 100644
--- a/autogen/agentchat/chat.py
+++ b/autogen/agentchat/chat.py
@@ -25,10 +25,12 @@ class ChatResult:
"""The chat history."""
summary: str = None
"""A summary obtained from the chat."""
- cost: tuple = None # (dict, dict) - (total_cost, actual_cost_with_cache)
- """The cost of the chat. a tuple of (total_cost, total_actual_cost), where total_cost is a
- dictionary of cost information, and total_actual_cost is a dictionary of information on
- the actual incurred cost with cache."""
+ cost: Dict[str, dict] = None # keys: "usage_including_cached_inference", "usage_excluding_cached_inference"
+ """The cost of the chat.
+ The value for each usage type is a dictionary containing cost information for that specific type.
+ - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference.
+ - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference".
+ """
human_input: List[str] = None
"""A list of human input solicited during the chat."""
diff --git a/autogen/agentchat/utils.py b/autogen/agentchat/utils.py
index eef374160..b32c2f5f0 100644
--- a/autogen/agentchat/utils.py
+++ b/autogen/agentchat/utils.py
@@ -1,5 +1,5 @@
import re
-from typing import Any, Callable, Dict, List, Tuple, Union
+from typing import Any, Callable, Dict, List, Union
from .agent import Agent
@@ -26,33 +26,46 @@ def consolidate_chat_info(chat_info, uniform_sender=None) -> None:
), "llm client must be set in either the recipient or sender when summary_method is reflection_with_llm."
-def gather_usage_summary(agents: List[Agent]) -> Tuple[Dict[str, any], Dict[str, any]]:
+def gather_usage_summary(agents: List[Agent]) -> Dict[Dict[str, Dict], Dict[str, Dict]]:
r"""Gather usage summary from all agents.
Args:
agents: (list): List of agents.
Returns:
- tuple: (total_usage_summary, actual_usage_summary)
+ dictionary: A dictionary containing two keys:
+ - "usage_including_cached_inference": Cost information on the total usage, including the tokens in cached inference.
+ - "usage_excluding_cached_inference": Cost information on the usage of tokens, excluding the tokens in cache. No larger than "usage_including_cached_inference".
Example:
```python
- total_usage_summary = {
- "total_cost": 0.0006090000000000001,
- "gpt-35-turbo": {
- "cost": 0.0006090000000000001,
- "prompt_tokens": 242,
- "completion_tokens": 123,
- "total_tokens": 365
+ {
+ "usage_including_cached_inference" : {
+ "total_cost": 0.0006090000000000001,
+ "gpt-35-turbo": {
+ "cost": 0.0006090000000000001,
+ "prompt_tokens": 242,
+ "completion_tokens": 123,
+ "total_tokens": 365
+ },
+ },
+
+ "usage_excluding_cached_inference" : {
+ "total_cost": 0.0006090000000000001,
+ "gpt-35-turbo": {
+ "cost": 0.0006090000000000001,
+ "prompt_tokens": 242,
+ "completion_tokens": 123,
+ "total_tokens": 365
+ },
}
}
```
Note:
- `actual_usage_summary` follows the same format.
- If none of the agents incurred any cost (not having a client), then the total_usage_summary and actual_usage_summary will be `{'total_cost': 0}`.
+ If none of the agents incurred any cost (not having a client), then the usage_including_cached_inference and usage_excluding_cached_inference will be `{'total_cost': 0}`.
"""
def aggregate_summary(usage_summary: Dict[str, Any], agent_summary: Dict[str, Any]) -> None:
@@ -69,15 +82,18 @@ def gather_usage_summary(agents: List[Agent]) -> Tuple[Dict[str, any], Dict[str,
usage_summary[model]["completion_tokens"] += data.get("completion_tokens", 0)
usage_summary[model]["total_tokens"] += data.get("total_tokens", 0)
- total_usage_summary = {"total_cost": 0}
- actual_usage_summary = {"total_cost": 0}
+ usage_including_cached_inference = {"total_cost": 0}
+ usage_excluding_cached_inference = {"total_cost": 0}
for agent in agents:
if getattr(agent, "client", None):
- aggregate_summary(total_usage_summary, agent.client.total_usage_summary)
- aggregate_summary(actual_usage_summary, agent.client.actual_usage_summary)
+ aggregate_summary(usage_including_cached_inference, agent.client.total_usage_summary)
+ aggregate_summary(usage_excluding_cached_inference, agent.client.actual_usage_summary)
- return total_usage_summary, actual_usage_summary
+ return {
+ "usage_including_cached_inference": usage_including_cached_inference,
+ "usage_excluding_cached_inference": usage_excluding_cached_inference,
+ }
def parse_tags_from_content(tag: str, content: Union[str, List[Dict[str, Any]]]) -> List[Dict[str, Dict[str, str]]]:
diff --git a/autogen/code_utils.py b/autogen/code_utils.py
index aa75756e0..5057c8615 100644
--- a/autogen/code_utils.py
+++ b/autogen/code_utils.py
@@ -41,7 +41,7 @@ logger = logging.getLogger(__name__)
def content_str(content: Union[str, List[Union[UserMessageTextContentPart, UserMessageImageContentPart]], None]) -> str:
- """Converts the `content` field of an OpenAI merssage into a string format.
+ """Converts the `content` field of an OpenAI message into a string format.
This function processes content that may be a string, a list of mixed text and image URLs, or None,
and converts it into a string. Text is directly appended to the result string, while image URLs are
diff --git a/notebook/agentchat_agentoptimizer.ipynb b/notebook/agentchat_agentoptimizer.ipynb
index dd5624458..924f1a557 100644
--- a/notebook/agentchat_agentoptimizer.ipynb
+++ b/notebook/agentchat_agentoptimizer.ipynb
@@ -459,7 +459,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.18"
+ "version": "3.9.13"
}
},
"nbformat": 4,
diff --git a/notebook/agentchat_auto_feedback_from_code_execution.ipynb b/notebook/agentchat_auto_feedback_from_code_execution.ipynb
index 834e1e7df..37584acdc 100644
--- a/notebook/agentchat_auto_feedback_from_code_execution.ipynb
+++ b/notebook/agentchat_auto_feedback_from_code_execution.ipynb
@@ -692,6 +692,7 @@
" file_content = \"No data found.\"\n",
" return \"Analyze the data and write a brief but engaging blog post. \\n Data: \\n\" + file_content\n",
"\n",
+ "\n",
"# followup of the previous question\n",
"chat_res = user_proxy.initiate_chat(\n",
" recipient=assistant,\n",
diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb
index 7334f2e36..457d58517 100644
--- a/notebook/agentchat_cost_token_tracking.ipynb
+++ b/notebook/agentchat_cost_token_tracking.ipynb
@@ -494,8 +494,8 @@
}
],
"source": [
- "total_usage_summary, actual_usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n",
- "total_usage_summary"
+ "usage_summary = gather_usage_summary([assistant, ai_user_proxy, user_proxy])\n",
+ "usage_summary[\"usage_including_cached_inference\"]"
]
}
],
diff --git a/notebook/agentchat_groupchat_stateflow.ipynb b/notebook/agentchat_groupchat_stateflow.ipynb
index 6205e1147..c824046f7 100644
--- a/notebook/agentchat_groupchat_stateflow.ipynb
+++ b/notebook/agentchat_groupchat_stateflow.ipynb
@@ -112,7 +112,6 @@
")\n",
"\n",
"\n",
- "\n",
"coder = autogen.AssistantAgent(\n",
" name=\"Retrieve_Action_1\",\n",
" llm_config=gpt4_config,\n",
diff --git a/notebook/agentchat_image_generation_capability.ipynb b/notebook/agentchat_image_generation_capability.ipynb
index 7c0c366a5..b5d298d7f 100644
--- a/notebook/agentchat_image_generation_capability.ipynb
+++ b/notebook/agentchat_image_generation_capability.ipynb
@@ -135,6 +135,7 @@
" return content[\"text\"].rstrip().endswith(\"TERMINATE\")\n",
" return False\n",
"\n",
+ "\n",
"def critic_agent() -> autogen.ConversableAgent:\n",
" return autogen.ConversableAgent(\n",
" name=\"critic\",\n",
diff --git a/test/agentchat/test_agent_usage.py b/test/agentchat/test_agent_usage.py
index 212c5513e..e31672cdd 100755
--- a/test/agentchat/test_agent_usage.py
+++ b/test/agentchat/test_agent_usage.py
@@ -1,21 +1,18 @@
#!/usr/bin/env python3 -m pytest
import io
+import os
+import sys
from contextlib import redirect_stdout
import pytest
-from conftest import skip_openai
from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST
import autogen
from autogen import AssistantAgent, UserProxyAgent, gather_usage_summary
-try:
- import openai
-except ImportError:
- skip = True
-else:
- skip = False or skip_openai
+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
+from conftest import skip_openai as skip # noqa: E402
@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")
@@ -62,11 +59,11 @@ def test_gathering():
"gpt-4": {"cost": 0.3, "prompt_tokens": 100, "completion_tokens": 200, "total_tokens": 300},
}
- total_usage, _ = gather_usage_summary([assistant1, assistant2, assistant3])
+ total_usage = gather_usage_summary([assistant1, assistant2, assistant3])
- assert round(total_usage["total_cost"], 8) == 0.6
- assert round(total_usage["gpt-35-turbo"]["cost"], 8) == 0.3
- assert round(total_usage["gpt-4"]["cost"], 8) == 0.3
+ assert round(total_usage["usage_including_cached_inference"]["total_cost"], 8) == 0.6
+ assert round(total_usage["usage_including_cached_inference"]["gpt-35-turbo"]["cost"], 8) == 0.3
+ assert round(total_usage["usage_including_cached_inference"]["gpt-4"]["cost"], 8) == 0.3
# test when agent doesn't have client
user_proxy = UserProxyAgent(
@@ -77,7 +74,10 @@ def test_gathering():
default_auto_reply="That's all. Thank you.",
)
- total_usage, acutal_usage = gather_usage_summary([user_proxy])
+ total_usage = gather_usage_summary([user_proxy])
+ total_usage_summary = total_usage["usage_including_cached_inference"]
+
+ print("Total usage summary:", total_usage_summary)
@pytest.mark.skipif(skip, reason="openai not installed OR requested to skip")