A programming framework for agentic AI
AgentOps Runtime Logging Implementation (#2682)
Braelyn Boynton85ad929f34e4
diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.pyindex aeea27cff..b434fc648 100644--- a/autogen/agentchat/conversable_agent.py+++ b/autogen/agentchat/conversable_agent.py@@ -31,7 +31,7 @@ from ..formatting_utils import coloredfrom ..function_utils import get_function_schema, load_basemodels_if_needed, serialize_to_strfrom ..io.base import IOStreamfrom ..oai.client import ModelClient, OpenAIWrapper-from ..runtime_logging import log_event, log_new_agent, logging_enabled+from ..runtime_logging import log_event, log_function_use, log_new_agent, logging_enabledfrom .agent import Agent, LLMAgentfrom .chat import ChatResult, a_initiate_chats, initiate_chatsfrom .utils import consolidate_chat_info, gather_usage_summary@@ -1357,9 +1357,7 @@ class ConversableAgent(LLMAgent):# TODO: #1143 handle token limit exceeded errorresponse = llm_client.create(- context=messages[-1].pop("context", None),- messages=all_messages,- cache=cache,+ context=messages[-1].pop("context", None), messages=all_messages, cache=cache, agent=self)extracted_response = llm_client.extract_text_or_completion_object(response)[0]@@ -2528,13 +2526,14 @@ class ConversableAgent(LLMAgent):@functools.wraps(func)def _wrapped_func(*args, **kwargs):retval = func(*args, **kwargs)-+ log_function_use(self, func, kwargs, retval)return serialize_to_str(retval)@load_basemodels_if_needed@functools.wraps(func)async def _a_wrapped_func(*args, **kwargs):retval = await func(*args, **kwargs)+ log_function_use(self, func, kwargs, retval)return serialize_to_str(retval)wrapped_func = _a_wrapped_func if inspect.iscoroutinefunction(func) else _wrapped_funcdiff --git a/autogen/logger/base_logger.py b/autogen/logger/base_logger.pyindex 7c35f8a50..c5c236fa4 100644--- a/autogen/logger/base_logger.py+++ b/autogen/logger/base_logger.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport sqlite3import uuidfrom abc import ABC, abstractmethod-from typing import TYPE_CHECKING, Any, Dict, List, Union+from typing import TYPE_CHECKING, Any, Callable, Dict, List, TypeVar, Unionfrom openai import AzureOpenAI, OpenAIfrom openai.types.chat import ChatCompletion@@ -11,6 +11,7 @@ from openai.types.chat import ChatCompletionif TYPE_CHECKING:from autogen import Agent, ConversableAgent, OpenAIWrapper+F = TypeVar("F", bound=Callable[..., Any])ConfigItem = Dict[str, Union[str, List[str]]]LLMConfig = Dict[str, Union[None, float, int, ConfigItem, List[ConfigItem]]]@@ -32,6 +33,7 @@ class BaseLogger(ABC):invocation_id: uuid.UUID,client_id: int,wrapper_id: int,+ source: Union[str, Agent],request: Dict[str, Union[float, str, List[Dict[str, str]]]],response: Union[str, ChatCompletion],is_cached: int,@@ -49,9 +51,10 @@ class BaseLogger(ABC):invocation_id (uuid): A unique identifier for the invocation to the OpenAIWrapper.create method callclient_id (int): A unique identifier for the underlying OpenAI client instancewrapper_id (int): A unique identifier for the OpenAIWrapper instance- request (dict): A dictionary representing the the request or call to the OpenAI client endpoint+ source (str or Agent): The source/creator of the event as a string name or an Agent instance+ request (dict): A dictionary representing the request or call to the OpenAI client endpointresponse (str or ChatCompletion): The response from OpenAI- is_chached (int): 1 if the response was a cache hit, 0 otherwise+ is_cached (int): 1 if the response was a cache hit, 0 otherwisecost(float): The cost for OpenAI responsestart_time (str): A string representing the moment the request was initiated"""@@ -104,6 +107,18 @@ class BaseLogger(ABC):"""...+ @abstractmethod+ def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[str, Any], returns: Any) -> None:+ """+ Log the use of a registered function (could be a tool)++ Args:+ source (str or Agent): The source/creator of the event as a string name or an Agent instance+ function (F): The function information+ args (dict): The function args to log+ returns (any): The return+ """+@abstractmethoddef stop(self) -> None:"""diff --git a/autogen/logger/file_logger.py b/autogen/logger/file_logger.pyindex fedcc134a..15b2c457e 100644--- a/autogen/logger/file_logger.py+++ b/autogen/logger/file_logger.py@@ -5,7 +5,7 @@ import loggingimport osimport threadingimport uuid-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, TypeVar, Unionfrom openai import AzureOpenAI, OpenAIfrom openai.types.chat import ChatCompletion@@ -21,9 +21,21 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+F = TypeVar("F", bound=Callable[..., Any])+__all__ = ("FileLogger",)+def safe_serialize(obj: Any) -> str:+ def default(o: Any) -> str:+ if hasattr(o, "to_json"):+ return str(o.to_json())+ else:+ return f"<<non-serializable: {type(o).__qualname__}>>"++ return json.dumps(obj, default=default)++class FileLogger(BaseLogger):def __init__(self, config: Dict[str, Any]):self.config = config@@ -59,6 +71,7 @@ class FileLogger(BaseLogger):invocation_id: uuid.UUID,client_id: int,wrapper_id: int,+ source: Union[str, Agent],request: Dict[str, Union[float, str, List[Dict[str, str]]]],response: Union[str, ChatCompletion],is_cached: int,@@ -69,6 +82,11 @@ class FileLogger(BaseLogger):Log a chat completion."""thread_id = threading.get_ident()+ source_name = None+ if isinstance(source, str):+ source_name = source+ else:+ source_name = source.nametry:log_data = json.dumps({@@ -82,6 +100,7 @@ class FileLogger(BaseLogger):"start_time": start_time,"end_time": get_current_ts(),"thread_id": thread_id,+ "source_name": source_name,})@@ -204,6 +223,29 @@ class FileLogger(BaseLogger):except Exception as e:self.logger.error(f"[file_logger] Failed to log event {e}")+ def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[str, Any], returns: Any) -> None:+ """+ Log a registered function(can be a tool) use from an agent or a string source.+ """+ thread_id = threading.get_ident()++ try:+ log_data = json.dumps(+ {+ "source_id": id(source),+ "source_name": str(source.name) if hasattr(source, "name") else source,+ "agent_module": source.__module__,+ "agent_class": source.__class__.__name__,+ "timestamp": get_current_ts(),+ "thread_id": thread_id,+ "input_args": safe_serialize(args),+ "returns": safe_serialize(returns),+ }+ )+ self.logger.info(log_data)+ except Exception as e:+ self.logger.error(f"[file_logger] Failed to log event {e}")+def get_connection(self) -> None:"""Method is intentionally left blank because there is no specific connection needed for the FileLogger."""passdiff --git a/autogen/logger/sqlite_logger.py b/autogen/logger/sqlite_logger.pyindex 48d33082f..4ebea32cc 100644--- a/autogen/logger/sqlite_logger.py+++ b/autogen/logger/sqlite_logger.py@@ -6,7 +6,7 @@ import osimport sqlite3import threadingimport uuid-from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Tuple, TypeVar, Unionfrom openai import AzureOpenAI, OpenAIfrom openai.types.chat import ChatCompletion@@ -25,6 +25,18 @@ lock = threading.Lock()__all__ = ("SqliteLogger",)+F = TypeVar("F", bound=Callable[..., Any])+++def safe_serialize(obj: Any) -> str:+ def default(o: Any) -> str:+ if hasattr(o, "to_json"):+ return str(o.to_json())+ else:+ return f"<<non-serializable: {type(o).__qualname__}>>"++ return json.dumps(obj, default=default)+class SqliteLogger(BaseLogger):schema_version = 1@@ -49,6 +61,7 @@ class SqliteLogger(BaseLogger):client_id INTEGER,wrapper_id INTEGER,session_id TEXT,+ source_name TEXT,request TEXT,response TEXT,is_cached INEGER,@@ -118,6 +131,18 @@ class SqliteLogger(BaseLogger):"""self._run_query(query=query)+ query = """+ CREATE TABLE IF NOT EXISTS function_calls (+ source_id INTEGER,+ source_name TEXT,+ function_name TEXT,+ args TEXT DEFAULT NULL,+ returns TEXT DEFAULT NULL,+ timestamp DATETIME DEFAULT CURRENT_TIMESTAMP+ );+ """+ self._run_query(query=query)+current_verion = self._get_current_db_version()if current_verion is None:self._run_query(@@ -192,6 +217,7 @@ class SqliteLogger(BaseLogger):invocation_id: uuid.UUID,client_id: int,wrapper_id: int,+ source: Union[str, Agent],request: Dict[str, Union[float, str, List[Dict[str, str]]]],response: Union[str, ChatCompletion],is_cached: int,@@ -208,10 +234,16 @@ class SqliteLogger(BaseLogger):else:response_messages = json.dumps(to_dict(response), indent=4)+ source_name = None+ if isinstance(source, str):+ source_name = source+ else:+ source_name = source.name+query = """INSERT INTO chat_completions (- invocation_id, client_id, wrapper_id, session_id, request, response, is_cached, cost, start_time, end_time- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ invocation_id, client_id, wrapper_id, session_id, request, response, is_cached, cost, start_time, end_time, source_name+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""args = (invocation_id,@@ -224,6 +256,7 @@ class SqliteLogger(BaseLogger):cost,start_time,end_time,+ source_name,)self._run_query(query=query, args=args)@@ -335,6 +368,24 @@ class SqliteLogger(BaseLogger):)self._run_query(query=query, args=args)+ def log_function_use(self, source: Union[str, Agent], function: F, args: Dict[str, Any], returns: Any) -> None:++ if self.con is None:+ return++ query = """+ INSERT INTO function_calls (source_id, source_name, function_name, args, returns, timestamp) VALUES (?, ?, ?, ?, ?, ?)+ """+ query_args: Tuple[Any, ...] = (+ id(source),+ source.name if hasattr(source, "name") else source,+ function.__name__,+ safe_serialize(args),+ safe_serialize(returns),+ get_current_ts(),+ )+ self._run_query(query=query, args=query_args)+def log_new_client(self, client: Union[AzureOpenAI, OpenAI, GeminiClient], wrapper: OpenAIWrapper, init_args: Dict[str, Any]) -> None:diff --git a/autogen/oai/client.py b/autogen/oai/client.pyindex 3aaf01736..4c1da7a39 100644--- a/autogen/oai/client.py+++ b/autogen/oai/client.py@@ -319,6 +319,7 @@ class OpenAIWrapper:"""A wrapper class for openai client."""extra_kwargs = {+ "agent","cache","cache_seed","filter_func",@@ -542,6 +543,7 @@ class OpenAIWrapper:Note that the cache argument overrides the legacy cache_seed argument: if this argument is provided,then the cache_seed argument is ignored. If this argument is not provided or None,then the cache_seed argument is used.+ - agent (AbstractAgent | None): The object responsible for creating a completion if an agent.- (Legacy) cache_seed (int | None) for using the DiskCache. Default to 41.An integer cache_seed is useful when implementing "controlled randomness" for the completion.None for no caching.@@ -589,6 +591,7 @@ class OpenAIWrapper:cache = extra_kwargs.get("cache")filter_func = extra_kwargs.get("filter_func")context = extra_kwargs.get("context")+ agent = extra_kwargs.get("agent")total_usage = Noneactual_usage = None@@ -626,6 +629,7 @@ class OpenAIWrapper:invocation_id=invocation_id,client_id=id(client),wrapper_id=id(self),+ agent=agent,request=params,response=response,is_cached=1,@@ -658,6 +662,7 @@ class OpenAIWrapper:invocation_id=invocation_id,client_id=id(client),wrapper_id=id(self),+ agent=agent,request=params,response=f"error_code:{error_code}, config {i} failed",is_cached=0,@@ -688,6 +693,7 @@ class OpenAIWrapper:invocation_id=invocation_id,client_id=id(client),wrapper_id=id(self),+ agent=agent,request=params,response=response,is_cached=0,diff --git a/autogen/runtime_logging.py b/autogen/runtime_logging.pyindex ffc741482..6ce207a2f 100644--- a/autogen/runtime_logging.py+++ b/autogen/runtime_logging.py@@ -3,7 +3,7 @@ from __future__ import annotationsimport loggingimport sqlite3import uuid-from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, TypeVar, Unionfrom openai import AzureOpenAI, OpenAIfrom openai.types.chat import ChatCompletion@@ -20,6 +20,8 @@ logger = logging.getLogger(__name__)autogen_logger = Noneis_logging = False+F = TypeVar("F", bound=Callable[..., Any])+def start(logger: Optional[BaseLogger] = None,@@ -56,6 +58,7 @@ def log_chat_completion(invocation_id: uuid.UUID,client_id: int,wrapper_id: int,+ agent: Union[str, Agent],request: Dict[str, Union[float, str, List[Dict[str, str]]]],response: Union[str, ChatCompletion],is_cached: int,@@ -67,7 +70,7 @@ def log_chat_completion(returnautogen_logger.log_chat_completion(- invocation_id, client_id, wrapper_id, request, response, is_cached, cost, start_time+ invocation_id, client_id, wrapper_id, agent, request, response, is_cached, cost, start_time)@@ -87,6 +90,14 @@ def log_event(source: Union[str, Agent], name: str, **kwargs: Dict[str, Any]) ->autogen_logger.log_event(source, name, **kwargs)+def log_function_use(agent: Union[str, Agent], function: F, args: Dict[str, Any], returns: any):+ if autogen_logger is None:+ logger.error("[runtime logging] log_function_use: autogen logger is None")+ return++ autogen_logger.log_function_use(agent, function, args, returns)++def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLMConfig, List[LLMConfig]]]) -> None:if autogen_logger is None:logger.error("[runtime logging] log_new_wrapper: autogen logger is None")diff --git a/notebook/agentchat_agentops.ipynb b/notebook/agentchat_agentops.ipynbnew file mode 100644index 000000000..293efa8e4--- /dev/null+++ b/notebook/agentchat_agentops.ipynb@@ -0,0 +1,533 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "id": "abb8a01d85d8b146",+ "metadata": {+ "collapsed": false+ },+ "source": [+ "# AgentOps"+ ]+ },+ {+ "cell_type": "markdown",+ "id": "a447802c88c8a240",+ "metadata": {},+ "source": [+ "\n",+ "\n",+ "[AgentOps](https://agentops.ai/?=autogen) provides session replays, metrics, and monitoring for AI agents.\n",+ "\n",+ "At a high level, AgentOps gives you the ability to monitor LLM calls, costs, latency, agent failures, multi-agent interactions, tool usage, session-wide statistics, and more. For more info, check out the [AgentOps Repo](https://github.com/AgentOps-AI/agentops).\n"+ ]+ },+ {+ "cell_type": "markdown",+ "id": "b354c068",+ "metadata": {},+ "source": [+ "### Dashboard\n",+ ""+ ]+ },+ {+ "cell_type": "markdown",+ "id": "38182a5296dceb34",+ "metadata": {},+ "source": [+ "## Adding AgentOps to an existing Autogen service.\n",+ "To get started, you'll need to install the AgentOps package and set an API key.\n",+ "\n",+ "AgentOps automatically configures itself when it's initialized. This means your agents will be tracked and logged to your AgentOps account right away."+ ]+ },+ {+ "cell_type": "markdown",+ "id": "8d9451f4",+ "metadata": {},+ "source": [+ "````{=mdx}\n",+ ":::info Requirements\n",+ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n",+ "\n",+ "```bash\n",+ "pip install pyautogen agentops\n",+ "```\n",+ "\n",+ "For more information, please refer to the [installation guide](/docs/installation/).\n",+ ":::\n",+ "````"+ ]+ },+ {+ "cell_type": "markdown",+ "id": "6be9e11620b0e8d6",+ "metadata": {},+ "source": [+ "### Set an API key\n",+ "\n",+ "By default, the AgentOps `init()` function will look for an environment variable named `AGENTOPS_API_KEY`. Alternatively, you can pass one in as an optional parameter.\n",+ "\n",+ "Create an account and API key at [AgentOps.ai](https://agentops.ai/)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "id": "f31a28d20a13b377",+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-05-31T22:48:27.679318Z",+ "start_time": "2024-05-31T22:48:26.192071Z"+ }+ },+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "π AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4\u001B[0m\u001B[0m\n"+ ]+ },+ {+ "data": {+ "text/plain": "UUID('8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4')"+ },+ "execution_count": 1,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "import agentops\n",+ "\n",+ "from autogen import ConversableAgent, UserProxyAgent, config_list_from_json\n",+ "\n",+ "agentops.init(api_key=\"7c94212b-b89d-47a6-a20c-23b2077d3226\") # or agentops.init(api_key=\"...\")"+ ]+ },+ {+ "cell_type": "markdown",+ "id": "4dd8f461ccd9cbef",+ "metadata": {},+ "source": [+ "Autogen will now start automatically tracking\n",+ "- LLM prompts and completions\n",+ "- Token usage and costs\n",+ "- Agent names and actions\n",+ "- Correspondence between agents\n",+ "- Tool usage\n",+ "- Errors"+ ]+ },+ {+ "cell_type": "markdown",+ "id": "712315c520536eb8",+ "metadata": {},+ "source": [+ "# Simple Chat Example"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "id": "66d68e66e9f4a677",+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-05-31T22:48:32.813123Z",+ "start_time": "2024-05-31T22:48:27.677564Z"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[33magent\u001B[0m (to user):\n",+ "\n",+ "How can I help you today?\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[33muser\u001B[0m (to agent):\n",+ "\n",+ "2+2\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33magent\u001B[0m (to user):\n",+ "\n",+ "2 + 2 equals 4.\n",+ "\n",+ "--------------------------------------------------------------------------------\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "π AgentOps: This run's cost $0.000960\n",+ "π AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=8bfaeed1-fd51-4c68-b3ec-276b1a3ce8a4\u001B[0m\u001B[0m\n"+ ]+ }+ ],+ "source": [+ "import agentops\n",+ "\n",+ "# When initializing AgentOps, you can pass in optional tags to help filter sessions\n",+ "agentops.init(tags=[\"simple-autogen-example\"])\n",+ "\n",+ "# Create the agent that uses the LLM.\n",+ "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n",+ "assistant = ConversableAgent(\"agent\", llm_config={\"config_list\": config_list})\n",+ "\n",+ "# Create the agent that represents the user in the conversation.\n",+ "user_proxy = UserProxyAgent(\"user\", code_execution_config=False)\n",+ "\n",+ "# Let the assistant start the conversation. It will end when the user types exit.\n",+ "assistant.initiate_chat(user_proxy, message=\"How can I help you today?\")\n",+ "\n",+ "# Close your AgentOps session to indicate that it completed.\n",+ "agentops.end_session(\"Success\")"+ ]+ },+ {+ "cell_type": "markdown",+ "id": "2217ed0f930cfcaa",+ "metadata": {},+ "source": [+ "You can view data on this run at [app.agentops.ai](https://app.agentops.ai). \n",+ "\n",+ "The dashboard will display LLM events for each message sent by each agent, including those made by the human user."+ ]+ },+ {+ "cell_type": "markdown",+ "source": [+ ""+ ],+ "metadata": {+ "collapsed": false+ },+ "id": "cbd689b0f5617013"+ },+ {+ "cell_type": "markdown",+ "id": "fd78f1a816276cb7",+ "metadata": {},+ "source": [+ "# Tool Example\n",+ "AgentOps also tracks when Autogen agents use tools. You can find more information on this example in [tool-use.ipynb](https://github.com/microsoft/autogen/blob/main/website/docs/tutorial/tool-use.ipynb)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "id": "3498aa6176c799ff",+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-05-31T22:48:35.808674Z",+ "start_time": "2024-05-31T22:48:32.813225Z"+ }+ },+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "π AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=880c206b-751e-4c23-9313-8684537fc04d\u001B[0m\u001B[0m\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "What is (1423 - 123) / 3 + (32 + 23) * 5?\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "\u001B[32m***** Suggested tool call (call_aINcGyo0Xkrh9g7buRuhyCz0): calculator *****\u001B[0m\n",+ "Arguments: \n",+ "{\n",+ " \"a\": 1423,\n",+ " \"b\": 123,\n",+ " \"operator\": \"-\"\n",+ "}\n",+ "\u001B[32m***************************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[32m***** Response from calling tool (call_aINcGyo0Xkrh9g7buRuhyCz0) *****\u001B[0m\n",+ "1300\n",+ "\u001B[32m**********************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "\u001B[32m***** Suggested tool call (call_prJGf8V0QVT7cbD91e0Fcxpb): calculator *****\u001B[0m\n",+ "Arguments: \n",+ "{\n",+ " \"a\": 1300,\n",+ " \"b\": 3,\n",+ " \"operator\": \"/\"\n",+ "}\n",+ "\u001B[32m***************************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[32m***** Response from calling tool (call_prJGf8V0QVT7cbD91e0Fcxpb) *****\u001B[0m\n",+ "433\n",+ "\u001B[32m**********************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "/Users/braelynboynton/Developer/agentops/autogen/autogen/agentchat/conversable_agent.py:2489: UserWarning: Function 'calculator' is being overridden.\n",+ " warnings.warn(f\"Function '{tool_sig['function']['name']}' is being overridden.\", UserWarning)\n",+ "/Users/braelynboynton/Developer/agentops/autogen/autogen/agentchat/conversable_agent.py:2408: UserWarning: Function 'calculator' is being overridden.\n",+ " warnings.warn(f\"Function '{name}' is being overridden.\", UserWarning)\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "\u001B[32m***** Suggested tool call (call_CUIgHRsySLjayDKuUphI1TGm): calculator *****\u001B[0m\n",+ "Arguments: \n",+ "{\n",+ " \"a\": 32,\n",+ " \"b\": 23,\n",+ " \"operator\": \"+\"\n",+ "}\n",+ "\u001B[32m***************************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[32m***** Response from calling tool (call_CUIgHRsySLjayDKuUphI1TGm) *****\u001B[0m\n",+ "55\n",+ "\u001B[32m**********************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "\u001B[32m***** Suggested tool call (call_L7pGtBLUf9V0MPL90BASyesr): calculator *****\u001B[0m\n",+ "Arguments: \n",+ "{\n",+ " \"a\": 55,\n",+ " \"b\": 5,\n",+ " \"operator\": \"*\"\n",+ "}\n",+ "\u001B[32m***************************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[32m***** Response from calling tool (call_L7pGtBLUf9V0MPL90BASyesr) *****\u001B[0m\n",+ "275\n",+ "\u001B[32m**********************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "\u001B[32m***** Suggested tool call (call_Ygo6p4XfcxRjkYBflhG3UVv6): calculator *****\u001B[0m\n",+ "Arguments: \n",+ "{\n",+ " \"a\": 433,\n",+ " \"b\": 275,\n",+ " \"operator\": \"+\"\n",+ "}\n",+ "\u001B[32m***************************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[32m***** Response from calling tool (call_Ygo6p4XfcxRjkYBflhG3UVv6) *****\u001B[0m\n",+ "708\n",+ "\u001B[32m**********************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "The result of the calculation is 708.\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "TERMINATE\n",+ "\n",+ "--------------------------------------------------------------------------------\n"+ ]+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "π AgentOps: This run's cost $0.001800\n",+ "π AgentOps: \u001B[34m\u001B[34mSession Replay: https://app.agentops.ai/drilldown?session_id=880c206b-751e-4c23-9313-8684537fc04d\u001B[0m\u001B[0m\n"+ ]+ }+ ],+ "source": [+ "from typing import Annotated, Literal\n",+ "\n",+ "from autogen import ConversableAgent, config_list_from_json, register_function\n",+ "\n",+ "agentops.start_session(tags=[\"autogen-tool-example\"])\n",+ "\n",+ "Operator = Literal[\"+\", \"-\", \"*\", \"/\"]\n",+ "\n",+ "\n",+ "def calculator(a: int, b: int, operator: Annotated[Operator, \"operator\"]) -> int:\n",+ " if operator == \"+\":\n",+ " return a + b\n",+ " elif operator == \"-\":\n",+ " return a - b\n",+ " elif operator == \"*\":\n",+ " return a * b\n",+ " elif operator == \"/\":\n",+ " return int(a / b)\n",+ " else:\n",+ " raise ValueError(\"Invalid operator\")\n",+ "\n",+ "\n",+ "config_list = config_list_from_json(env_or_file=\"OAI_CONFIG_LIST\")\n",+ "\n",+ "# Create the agent that uses the LLM.\n",+ "assistant = ConversableAgent(\n",+ " name=\"Assistant\",\n",+ " system_message=\"You are a helpful AI assistant. \"\n",+ " \"You can help with simple calculations. \"\n",+ " \"Return 'TERMINATE' when the task is done.\",\n",+ " llm_config={\"config_list\": config_list},\n",+ ")\n",+ "\n",+ "# The user proxy agent is used for interacting with the assistant agent\n",+ "# and executes tool calls.\n",+ "user_proxy = ConversableAgent(\n",+ " name=\"User\",\n",+ " llm_config=False,\n",+ " is_termination_msg=lambda msg: msg.get(\"content\") is not None and \"TERMINATE\" in msg[\"content\"],\n",+ " human_input_mode=\"NEVER\",\n",+ ")\n",+ "\n",+ "assistant.register_for_llm(name=\"calculator\", description=\"A simple calculator\")(calculator)\n",+ "user_proxy.register_for_execution(name=\"calculator\")(calculator)\n",+ "\n",+ "# Register the calculator function to the two agents.\n",+ "register_function(\n",+ " calculator,\n",+ " caller=assistant, # The assistant agent can suggest calls to the calculator.\n",+ " executor=user_proxy, # The user proxy agent can execute the calculator calls.\n",+ " name=\"calculator\", # By default, the function name is used as the tool name.\n",+ " description=\"A simple calculator\", # A description of the tool.\n",+ ")\n",+ "\n",+ "# Let the assistant start the conversation. It will end when the user types exit.\n",+ "user_proxy.initiate_chat(assistant, message=\"What is (1423 - 123) / 3 + (32 + 23) * 5?\")\n",+ "\n",+ "agentops.end_session(\"Success\")"+ ]+ },+ {+ "cell_type": "markdown",+ "id": "2b4edf8e70d17267",+ "metadata": {},+ "source": [+ "You can see your run in action at [app.agentops.ai](https://app.agentops.ai). In this example, the AgentOps dashboard will show:\n",+ "- Agents talking to each other\n",+ "- Each use of the `calculator` tool\n",+ "- Each call to OpenAI for LLM use"+ ]+ },+ {+ "cell_type": "markdown",+ "source": [+ ""+ ],+ "metadata": {+ "collapsed": false+ },+ "id": "a922a52ab5fce31"+ }+ ],+ "metadata": {+ "front_matter": {+ "description": "Use AgentOps to simplify the development process and monitor your agents in production.",+ "tags": [+ "monitoring",+ "debugging"+ ]+ },+ "kernelspec": {+ "display_name": "Python 3 (ipykernel)",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.5"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 5+}diff --git a/notebook/agentchat_auto_feedback_from_code_execution.ipynb b/notebook/agentchat_auto_feedback_from_code_execution.ipynbindex bf784889d..6ea6f662b 100644--- a/notebook/agentchat_auto_feedback_from_code_execution.ipynb+++ b/notebook/agentchat_auto_feedback_from_code_execution.ipynb@@ -40,7 +40,7 @@" filter_dict={\"tags\": [\"gpt-4\"]}, # comment out to get all\n",")\n","# When using a single openai endpoint, you can use the following:\n",- "# config_list = [{\"model\": \"gpt-4\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]\n"+ "# config_list = [{\"model\": \"gpt-4\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]"]},{diff --git a/notebook/agentchat_logging.ipynb b/notebook/agentchat_logging.ipynbindex 7eb4138b4..eb5a6e752 100644--- a/notebook/agentchat_logging.ipynb+++ b/notebook/agentchat_logging.ipynb@@ -25,12 +25,12 @@"output_type": "stream","text": ["Logging session ID: 6e08f3e0-392b-434e-8b69-4ab36c4fcf99\n",- "\u001b[33muser_proxy\u001b[0m (to assistant):\n",+ "\u001B[33muser_proxy\u001B[0m (to assistant):\n","\n","What is the height of the Eiffel Tower? Only respond with the answer and terminate\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[33massistant\u001b[0m (to user_proxy):\n",+ "\u001B[33massistant\u001B[0m (to user_proxy):\n","\n","The height of the Eiffel Tower is approximately 330 meters.\n","\n",@@ -313,12 +313,12 @@"output_type": "stream","text": ["Logging session ID: ed493ebf-d78e-49f0-b832-69557276d557\n",- "\u001b[33muser_proxy\u001b[0m (to assistant):\n",+ "\u001B[33muser_proxy\u001B[0m (to assistant):\n","\n","What is the height of the Eiffel Tower? Only respond with the answer and terminate\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[33massistant\u001b[0m (to user_proxy):\n",+ "\u001B[33massistant\u001B[0m (to user_proxy):\n","\n","The height of the Eiffel Tower is 330 meters.\n","TERMINATE\n",@@ -328,7 +328,6 @@}],"source": [- "\n","import pandas as pd\n","\n","import autogen\n",diff --git a/test/agentchat/test_agent_file_logging.py b/test/agentchat/test_agent_file_logging.pyindex 9b930ba4a..1818b85fa 100644--- a/test/agentchat/test_agent_file_logging.py+++ b/test/agentchat/test_agent_file_logging.py@@ -3,6 +3,7 @@ import osimport sysimport tempfileimport uuid+from typing import Any, Callableimport pytest@@ -19,6 +20,10 @@ from autogen.logger.file_logger import FileLoggeris_windows = sys.platform.startswith("win")+def dummy_function(param1: str, param2: int) -> Any:+ return param1 * param2++@pytest.mark.skipif(is_windows, reason="Skipping file logging tests on Windows")@pytest.fixturedef logger() -> FileLogger:@@ -49,8 +54,19 @@ def test_log_chat_completion(logger: FileLogger):is_cached = 0cost = 0.5start_time = "2024-05-06 15:20:21.263231"-- logger.log_chat_completion(invocation_id, client_id, wrapper_id, request, response, is_cached, cost, start_time)+ agent = autogen.AssistantAgent(name="TestAgent", code_execution_config=False)++ logger.log_chat_completion(+ invocation_id=invocation_id,+ client_id=client_id,+ wrapper_id=wrapper_id,+ request=request,+ response=response,+ is_cached=is_cached,+ cost=cost,+ start_time=start_time,+ source=agent,+ )with open(logger.log_file, "r") as f:lines = f.readlines()@@ -63,6 +79,26 @@ def test_log_chat_completion(logger: FileLogger):assert log_data["is_cached"] == is_cachedassert log_data["cost"] == costassert log_data["start_time"] == start_time+ assert log_data["source_name"] == "TestAgent"+ assert isinstance(log_data["thread_id"], int)++[email protected](is_windows, reason="Skipping file logging tests on Windows")+def test_log_function_use(logger: FileLogger):+ source = autogen.AssistantAgent(name="TestAgent", code_execution_config=False)+ func: Callable[[str, int], Any] = dummy_function+ args = {"foo": "bar"}+ returns = True++ logger.log_function_use(source=source, function=func, args=args, returns=returns)++ with open(logger.log_file, "r") as f:+ lines = f.readlines()+ assert len(lines) == 1+ log_data = json.loads(lines[0])+ assert log_data["source_name"] == "TestAgent"+ assert log_data["input_args"] == json.dumps(args)+ assert log_data["returns"] == json.dumps(returns)assert isinstance(log_data["thread_id"], int)diff --git a/test/test_logging.py b/test/test_logging.pyindex d824cc914..c6f7a182c 100644--- a/test/test_logging.py+++ b/test/test_logging.py@@ -1,6 +1,7 @@import jsonimport sqlite3import uuid+from typing import Any, Callablefrom unittest.mock import Mock, patchimport pytest@@ -61,6 +62,11 @@ SAMPLE_CHAT_RESPONSE = json.loads(""")++def dummy_function(param1: str, param2: int) -> Any:+ return param1 * param2++###############################################################@@ -84,6 +90,7 @@ def get_sample_chat_completion(response):"is_cached": 0,"cost": 0.347,"start_time": get_current_ts(),+ "agent": autogen.AssistantAgent(name="TestAgent", code_execution_config=False),}@@ -103,7 +110,7 @@ def test_log_completion(response, expected_logged_response, db_connection):query = """SELECT invocation_id, client_id, wrapper_id, request, response, is_cached,- cost, start_time FROM chat_completions+ cost, start_time, source_name FROM chat_completions"""for row in cur.execute(query):@@ -115,6 +122,28 @@ def test_log_completion(response, expected_logged_response, db_connection):assert row["is_cached"] == sample_completion["is_cached"]assert row["cost"] == sample_completion["cost"]assert row["start_time"] == sample_completion["start_time"]+ assert row["source_name"] == "TestAgent"+++def test_log_function_use(db_connection):+ cur = db_connection.cursor()++ source = autogen.AssistantAgent(name="TestAgent", code_execution_config=False)+ func: Callable[[str, int], Any] = dummy_function+ args = {"foo": "bar"}+ returns = True++ autogen.runtime_logging.log_function_use(agent=source, function=func, args=args, returns=returns)++ query = """+ SELECT source_id, source_name, function_name, args, returns, timestamp+ FROM function_calls+ """++ for row in cur.execute(query):+ assert row["source_name"] == "TestAgent"+ assert row["args"] == json.dumps(args)+ assert row["returns"] == json.dumps(returns)def test_log_new_agent(db_connection):diff --git a/website/docs/Examples.md b/website/docs/Examples.mdindex 5062cc5de..2ec83d1e0 100644--- a/website/docs/Examples.md+++ b/website/docs/Examples.md@@ -100,6 +100,9 @@ Links to notebook examples:- Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb)- Automatically Build Multi-agent System from Agent Library - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_agent_library.ipynb)+### Observability+- Track LLM calls, tool usage, actions and errors using AgentOps - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentops.ipynb)+## Enhanced Inferences### Utilitiesdiff --git a/website/docs/ecosystem/agentops.md b/website/docs/ecosystem/agentops.mdnew file mode 100644index 000000000..76995b6eb--- /dev/null+++ b/website/docs/ecosystem/agentops.md@@ -0,0 +1,83 @@+# AgentOps ποΈ++++[AgentOps](https://agentops.ai/?=autogen) provides session replays, metrics, and monitoring for agents.++At a high level, AgentOps gives you the ability to monitor LLM calls, costs, latency, agent failures, multi-agent interactions, tool usage, session-wide statistics, and more. For more info, check out the [AgentOps Repo](https://github.com/AgentOps-AI/agentops).++<details open>+ <summary>Agent Dashboard</summary>+ <a href="https://app.agentops.ai?ref=gh">+ <img src="https://github.com/AgentOps-AI/agentops/assets/14807319/158e082a-9a7d-49b7-9b41-51a49a1f7d3d" style="width: 90%;" alt="Agent Dashboard"/>+ </a>+</details>++<details>+ <summary>Session Analytics</summary>+ <a href="https://app.agentops.ai?ref=gh">+ <img src="https://github.com/AgentOps-AI/agentops/assets/14807319/d7228019-1488-40d3-852f-a61e998658ad" style="width: 90%;" alt="Session Analytics"/>+ </a>+</details>++<details>+ <summary>Session Replays</summary>+ <a href="https://app.agentops.ai?ref=gh">+ <img src="https://github.com/AgentOps-AI/agentops/assets/14807319/561d59f3-c441-4066-914b-f6cfe32a598c" style="width: 90%;" alt="Session Replays"/>+ </a>+</details>+++## Installation++AgentOps works seamlessly with applications built using Autogen.++1. **Install AgentOps**+```bash+pip install agentops+```++2. **Create an API Key:**+Create a user API key here: [Create API Key](https://app.agentops.ai/account)++3. **Configure Your Environment:**+Add your API key to your environment variables++```+AGENTOPS_API_KEY=<YOUR_AGENTOPS_API_KEY>+```++4. **Initialize AgentOps**++To start tracking all available data on Autogen runs, simply add two lines of code before implementing Autogen.++```python+import agentops+agentops.init() # Or: agentops.init(api_key="your-api-key-here")+```++After initializing AgentOps, Autogen will now start automatically tracking your agent runs.++## Features++- **LLM Costs**: Track spend with foundation model providers+- **Replay Analytics**: Watch step-by-step agent execution graphs+- **Recursive Thought Detection**: Identify when agents fall into infinite loops+- **Custom Reporting:** Create custom analytics on agent performance+- **Analytics Dashboard:** Monitor high level statistics about agents in development and production+- **Public Model Testing**: Test your agents against benchmarks and leaderboards+- **Custom Tests:** Run your agents against domain specific tests+- **Time Travel Debugging**: Save snapshots of session states to rewind and replay agent runs from chosen checkpoints.+- **Compliance and Security**: Create audit logs and detect potential threats such as profanity and PII leaks+- **Prompt Injection Detection**: Identify potential code injection and secret leaks++## Autogen + AgentOps examples+* [AgentChat with AgentOps Notebook](/docs/notebooks/agentchat_agentops)+* [More AgentOps Examples](https://docs.agentops.ai/v1/quickstart)++## Extra links++- [π¦ Twitter](https://twitter.com/agentopsai/)+- [π’ Discord](https://discord.gg/JHPt4C7r)+- [ποΈ AgentOps Dashboard](https://app.agentops.ai/ref?=autogen)+- [π Documentation](https://docs.agentops.ai/introduction)diff --git a/website/docs/topics/llm-observability.md b/website/docs/topics/llm-observability.mdnew file mode 100644index 000000000..6a95d185f--- /dev/null+++ b/website/docs/topics/llm-observability.md@@ -0,0 +1,42 @@+# LLM Observability++AutoGen supports advanced LLM observability and monitoring through built-in logging and partner providers.++## What is LLM Observability+AI agent observability is the ability to monitor, measure, and understand the internal states and behaviors of AI agent systems.+Observability is crucial for ensuring transparency, reliability, and accountability in your agent systems.+++## Development++### Agent Development in Terminal is Limited+- Lose track of what your agents did in between executions+- Parsing through terminal output searching for LLM completions+- Printing βtool calledβ++### Agent Development Dashboards Enable More+- Visual dashboard so you can see what your agents did in human-readable format+- LLM calls are magically recorded - prompt, completion, timestamps for each - with one line of code+- Agents and their events (including tool calls) are recorded with one more line of code+- Errors are magically associated to its causal event+- Record any other events to your session with two more lines of code+- Tons of other useful data if youβre developing with supported agent frameworks: SDK version++## Compliance++Observability and monitoring is critical to ensure AI agent systems adhere to laws and regulations in industries like finance and healthcare, preventing violations such as data breaches and privacy issues.++- Insights into AI decision-making, allowing organizations to explain outcomes and build trust with stakeholders.+- Helps detect anomalies and unintended behaviors early, mitigating operational, financial, and reputational risks.+- Ensures compliance with data privacy regulations, preventing unauthorized access and misuse of sensitive information.+- Quick identification and response to compliance violations, supporting incident analysis and prevention.++## Available Observability Integrations++### Logging+- Autogen SQLite and File Logger - [Tutorial](/docs/notebooks/agentchat_logging)++### Full-Service Partners+Autogen is currently partnered with [AgentOps](https://agentops.ai) for seamless observability integration.++[Learn how to install AgentOps](/docs/notebooks/agentchat_agentops)diff --git a/website/docs/topics/llm_configuration.ipynb b/website/docs/topics/llm_configuration.ipynbindex 51abf1f46..c0a1b7e74 100644--- a/website/docs/topics/llm_configuration.ipynb+++ b/website/docs/topics/llm_configuration.ipynb@@ -279,6 +279,7 @@" def __deepcopy__(self, memo):\n"," return self\n","\n",+ "\n","config_list = [\n"," {\n"," \"model\": \"my-gpt-4-deployment\",\n",diff --git a/website/docs/tutorial/tool-use.ipynb b/website/docs/tutorial/tool-use.ipynbindex 12d0f5a80..b5e215592 100644--- a/website/docs/tutorial/tool-use.ipynb+++ b/website/docs/tutorial/tool-use.ipynb@@ -39,8 +39,13 @@},{"cell_type": "code",- "execution_count": 1,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:48.858694Z",+ "start_time": "2024-04-25T01:49:48.854420Z"+ }+ },"outputs": [],"source": ["from typing import Annotated, Literal\n",@@ -91,8 +96,24 @@{"cell_type": "code","execution_count": null,- "metadata": {},- "outputs": [],+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:48.946697Z",+ "start_time": "2024-04-25T01:49:48.857869Z"+ }+ },+ "outputs": [+ {+ "data": {+ "text/plain": [+ "<function __main__.calculator(a: int, b: int, operator: Annotated[Literal['+', '-', '*', '/'], 'operator']) -> int>"+ ]+ },+ "execution_count": 4,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["import os\n","\n",@@ -160,8 +181,13 @@},{"cell_type": "code",- "execution_count": 3,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:48.953345Z",+ "start_time": "2024-04-25T01:49:48.947026Z"+ }+ },"outputs": [],"source": ["from autogen import register_function\n",@@ -189,124 +215,128 @@},{"cell_type": "code",- "execution_count": 4,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:57.947530Z",+ "start_time": "2024-04-25T01:49:48.953943Z"+ }+ },"outputs": [{"name": "stdout","output_type": "stream","text": [- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n","What is (44232 + 13312 / (232 - 32)) * 5?\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "\u001b[32m***** Suggested tool Call (call_bACquf0OreI0VHh7rWiP6ZE7): calculator *****\u001b[0m\n",+ "\u001B[32m***** Suggested tool call (call_4rElPoLggOYJmkUutbGaSTX1): calculator *****\u001B[0m\n","Arguments: \n","{\n",- " \"a\": 13312,\n",- " \"b\": 232 - 32,\n",- " \"operator\": \"/\"\n",+ " \"a\": 232,\n",+ " \"b\": 32,\n",+ " \"operator\": \"-\"\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_bACquf0OreI0VHh7rWiP6ZE7\" *****\u001b[0m\n",- "Error: Expecting ',' delimiter: line 1 column 26 (char 25)\n",- " You argument should follow json format.\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_4rElPoLggOYJmkUutbGaSTX1) *****\u001B[0m\n",+ "200\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "\u001b[32m***** Suggested tool Call (call_2c0H5gzX9SWsJ05x7nEOVbav): calculator *****\u001b[0m\n",+ "\u001B[32m***** Suggested tool call (call_SGtr8tK9A4iOCJGdCqkKR2Ov): calculator *****\u001B[0m\n","Arguments: \n","{\n"," \"a\": 13312,\n"," \"b\": 200,\n"," \"operator\": \"/\"\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[35m\n",- ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_2c0H5gzX9SWsJ05x7nEOVbav\" *****\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_SGtr8tK9A4iOCJGdCqkKR2Ov) *****\u001B[0m\n","66\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "\u001b[32m***** Suggested tool Call (call_ioceLhuKMpfU131E7TSQ8wCD): calculator *****\u001b[0m\n",+ "\u001B[32m***** Suggested tool call (call_YsR95CM1Ice2GZ7ZoStYXI6M): calculator *****\u001B[0m\n","Arguments: \n","{\n"," \"a\": 44232,\n"," \"b\": 66,\n"," \"operator\": \"+\"\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[35m\n",- ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_ioceLhuKMpfU131E7TSQ8wCD\" *****\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_YsR95CM1Ice2GZ7ZoStYXI6M) *****\u001B[0m\n","44298\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "\u001b[32m***** Suggested tool Call (call_0rhx9vrbigcbqLssKLh4sS7j): calculator *****\u001b[0m\n",+ "\u001B[32m***** Suggested tool call (call_oqZn4rTjyvXYcmjAXkvVaJm1): calculator *****\u001B[0m\n","Arguments: \n","{\n"," \"a\": 44298,\n"," \"b\": 5,\n"," \"operator\": \"*\"\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[35m\n",- ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_0rhx9vrbigcbqLssKLh4sS7j\" *****\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_oqZn4rTjyvXYcmjAXkvVaJm1) *****\u001B[0m\n","221490\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",- "\n",- "The result of the calculation (44232 + 13312 / (232 - 32)) * 5 is 221490. \n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "TERMINATE\n",+ "The result of the calculation is 221490. TERMINATE\n","\n","--------------------------------------------------------------------------------\n"]@@ -325,8 +355,13 @@},{"cell_type": "code",- "execution_count": 5,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:57.956063Z",+ "start_time": "2024-04-25T01:49:57.948882Z"+ }+ },"outputs": [{"data": {@@ -334,7 +369,7 @@"221490"]},- "execution_count": 5,+ "execution_count": 7,"metadata": {},"output_type": "execute_result"}@@ -369,8 +404,13 @@},{"cell_type": "code",- "execution_count": 6,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:57.956696Z",+ "start_time": "2024-04-25T01:49:57.952767Z"+ }+ },"outputs": [{"data": {@@ -387,7 +427,7 @@" 'required': ['a', 'b', 'operator']}}}]"]},- "execution_count": 6,+ "execution_count": 8,"metadata": {},"output_type": "execute_result"}@@ -417,8 +457,13 @@},{"cell_type": "code",- "execution_count": 33,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:57.965069Z",+ "start_time": "2024-04-25T01:49:57.958274Z"+ }+ },"outputs": [],"source": ["from pydantic import BaseModel, Field\n",@@ -459,8 +504,24 @@{"cell_type": "code","execution_count": null,- "metadata": {},- "outputs": [],+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:57.990811Z",+ "start_time": "2024-04-25T01:49:57.962315Z"+ }+ },+ "outputs": [+ {+ "data": {+ "text/plain": [+ "<function __main__.calculator(input: typing.Annotated[__main__.CalculatorInput, 'Input to the calculator.']) -> int>"+ ]+ },+ "execution_count": 10,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["assistant.register_for_llm(name=\"calculator\", description=\"A calculator tool that accepts nested expression as input\")(\n"," calculator\n",@@ -477,8 +538,13 @@},{"cell_type": "code",- "execution_count": 28,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:49:57.991342Z",+ "start_time": "2024-04-25T01:49:57.972554Z"+ }+ },"outputs": [{"data": {@@ -504,7 +570,7 @@" 'required': ['input']}}}]"]},- "execution_count": 28,+ "execution_count": 11,"metadata": {},"output_type": "execute_result"}@@ -522,159 +588,192 @@},{"cell_type": "code",- "execution_count": 32,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:50:17.808416Z",+ "start_time": "2024-04-25T01:49:57.975143Z"+ }+ },"outputs": [{"name": "stdout","output_type": "stream","text": [- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n","What is (1423 - 123) / 3 + (32 + 23) * 5?\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "\u001b[32m***** Suggested tool Call (call_t9By3vewGRoSLWsvdTR7p8Zo): calculator *****\u001b[0m\n",+ "\u001B[32m***** Suggested tool call (call_Uu4diKtxlTfkwXuY6MmJEb4E): calculator *****\u001B[0m\n","Arguments: \n","{\n",- " \"input\": {\n",- " \"a\": 1423,\n",- " \"b\": 123,\n",- " \"operator\": \"-\"\n",- " }\n",+ " \"input\": {\n",+ " \"a\": (1423 - 123) / 3,\n",+ " \"b\": (32 + 23) * 5,\n",+ " \"operator\": \"+\"\n",+ " }\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[35m\n",- ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_t9By3vewGRoSLWsvdTR7p8Zo\" *****\u001b[0m\n",- "1300\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_Uu4diKtxlTfkwXuY6MmJEb4E) *****\u001B[0m\n",+ "Error: Expecting value: line 1 column 29 (char 28)\n",+ " You argument should follow json format.\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "I apologize for the confusion, I seem to have made a mistake. Let me recalculate the expression properly.\n","\n",- "\u001b[32m***** Suggested tool Call (call_rhecyhVCo0Y8HPL193xOUPE6): calculator *****\u001b[0m\n",+ "First, we need to do the calculations within the brackets. So, calculating (1423 - 123), (32 + 23), and then performing remaining operations.\n",+ "\u001B[32m***** Suggested tool call (call_mx3M3fNOwikFNoqSojDH1jIr): calculator *****\u001B[0m\n","Arguments: \n","{\n",- " \"input\": {\n",- " \"a\": 1300,\n",- " \"b\": 3,\n",- " \"operator\": \"/\"\n",- " }\n",+ " \"input\": {\n",+ " \"a\": 1423,\n",+ " \"b\": 123,\n",+ " \"operator\": \"-\"\n",+ " }\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[35m\n",- ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_rhecyhVCo0Y8HPL193xOUPE6\" *****\u001b[0m\n",- "433\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_mx3M3fNOwikFNoqSojDH1jIr) *****\u001B[0m\n",+ "1300\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "\u001b[32m***** Suggested tool Call (call_zDpq9J5MYAsL7uS8cobOwa7S): calculator *****\u001b[0m\n",+ "\u001B[32m***** Suggested tool call (call_hBAL2sYi6Y5ZtTHCNPCmxdN3): calculator *****\u001B[0m\n","Arguments: \n","{\n",- " \"input\": {\n",- " \"a\": 32,\n",- " \"b\": 23,\n",- " \"operator\": \"+\"\n",- " }\n",+ " \"input\": {\n",+ " \"a\": 32,\n",+ " \"b\": 23,\n",+ " \"operator\": \"+\"\n",+ " }\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[35m\n",- ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_zDpq9J5MYAsL7uS8cobOwa7S\" *****\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_hBAL2sYi6Y5ZtTHCNPCmxdN3) *****\u001B[0m\n","55\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "\u001b[32m***** Suggested tool Call (call_mjDuVMojOIdaxmvDUIF4QtVi): calculator *****\u001b[0m\n",+ "\u001B[32m***** Suggested tool call (call_wO3AP7EDeJvsVLCpvv5LohUa): calculator *****\u001B[0m\n","Arguments: \n","{\n",- " \"input\": {\n",- " \"a\": 55,\n",- " \"b\": 5,\n",- " \"operator\": \"*\"\n",- " }\n",+ " \"input\": {\n",+ " \"a\": 1300,\n",+ " \"b\": 3,\n",+ " \"operator\": \"/\"\n",+ " }\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[35m\n",- ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_mjDuVMojOIdaxmvDUIF4QtVi\" *****\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_wO3AP7EDeJvsVLCpvv5LohUa) *****\u001B[0m\n",+ "433\n",+ "\u001B[32m**********************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n",+ "\n",+ "\u001B[32m***** Suggested tool call (call_kQ2hDhqem8BHNlaHaE9ezvvQ): calculator *****\u001B[0m\n",+ "Arguments: \n",+ "{\n",+ " \"input\": {\n",+ " \"a\": 55,\n",+ " \"b\": 5,\n",+ " \"operator\": \"*\"\n",+ " }\n",+ "}\n",+ "\u001B[32m***************************************************************************\u001B[0m\n",+ "\n",+ "--------------------------------------------------------------------------------\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n",+ "\n",+ "\u001B[32m***** Response from calling tool (call_kQ2hDhqem8BHNlaHaE9ezvvQ) *****\u001B[0m\n","275\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",- "\u001b[32m***** Suggested tool Call (call_hpirkAGKOewZstsDOxL2sYNW): calculator *****\u001b[0m\n",+ "\u001B[32m***** Suggested tool call (call_1FLDUdvAZmjlSD7g5GFFJOpO): calculator *****\u001B[0m\n","Arguments: \n","{\n",- " \"input\": {\n",- " \"a\": 433,\n",- " \"b\": 275,\n",- " \"operator\": \"+\"\n",- " }\n",+ " \"input\": {\n",+ " \"a\": 433,\n",+ " \"b\": 275,\n",+ " \"operator\": \"+\"\n",+ " }\n","}\n",- "\u001b[32m***************************************************************************\u001b[0m\n",+ "\u001B[32m***************************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[35m\n",- ">>>>>>>> EXECUTING FUNCTION calculator...\u001b[0m\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[35m\n",+ ">>>>>>>> EXECUTING FUNCTION calculator...\u001B[0m\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[33mUser\u001b[0m (to Assistant):\n",+ "\u001B[33mUser\u001B[0m (to Assistant):\n","\n",- "\u001b[32m***** Response from calling tool \"call_hpirkAGKOewZstsDOxL2sYNW\" *****\u001b[0m\n",+ "\u001B[32m***** Response from calling tool (call_1FLDUdvAZmjlSD7g5GFFJOpO) *****\u001B[0m\n","708\n",- "\u001b[32m**********************************************************************\u001b[0m\n",+ "\u001B[32m**********************************************************************\u001B[0m\n","\n","--------------------------------------------------------------------------------\n",- "\u001b[31m\n",- ">>>>>>>> USING AUTO REPLY...\u001b[0m\n",- "\u001b[33mAssistant\u001b[0m (to User):\n",- "\n",- "The result of the calculation is 708. \n",+ "\u001B[31m\n",+ ">>>>>>>> USING AUTO REPLY...\u001B[0m\n",+ "\u001B[33mAssistant\u001B[0m (to User):\n","\n",+ "The calculation result of the expression (1423 - 123) / 3 + (32 + 23) * 5 is 708. Let's proceed to the next task.\n","TERMINATE\n","\n","--------------------------------------------------------------------------------\n"@@ -694,8 +793,13 @@},{"cell_type": "code",- "execution_count": 31,- "metadata": {},+ "execution_count": null,+ "metadata": {+ "ExecuteTime": {+ "end_time": "2024-04-25T01:50:17.818095Z",+ "start_time": "2024-04-25T01:50:17.808502Z"+ }+ },"outputs": [{"data": {@@ -703,7 +807,7 @@"708"]},- "execution_count": 31,+ "execution_count": 13,"metadata": {},"output_type": "execute_result"}