Skip to content

A programming framework for agentic AI

Logging (#1146)

cheng-tan4ccff54dbe14
diff --git a/autogen/agentchat/assistant_agent.py b/autogen/agentchat/assistant_agent.py
index bdec0fef6..845a937a7 100644
--- a/autogen/agentchat/assistant_agent.py
+++ b/autogen/agentchat/assistant_agent.py
@@ -1,6 +1,7 @@
from typing import Callable, Dict, Literal, Optional, Union
from .conversable_agent import ConversableAgent
+from autogen.runtime_logging import logging_enabled, log_new_agent
class AssistantAgent(ConversableAgent):
@@ -67,6 +68,8 @@ Reply "TERMINATE" in the end when everything is done.
description=description,
**kwargs,
)
+ if logging_enabled():
+ log_new_agent(self, locals())
# Update the provided description if None, and we are using the default system_message,
# then use the default description.
diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py
index eafa0854f..29006021e 100644
--- a/autogen/agentchat/conversable_agent.py
+++ b/autogen/agentchat/conversable_agent.py
@@ -14,6 +14,7 @@ from ..coding.base import CodeExecutor
from ..coding.factory import CodeExecutorFactory
from ..oai.client import OpenAIWrapper, ModelClient
+from ..runtime_logging import logging_enabled, log_new_agent
from ..cache.cache import Cache
from ..code_utils import (
UNKNOWN,
@@ -145,6 +146,9 @@ class ConversableAgent(LLMAgent):
self.llm_config.update(llm_config)
self.client = OpenAIWrapper(**self.llm_config)
+ if logging_enabled():
+ log_new_agent(self, locals())
+
# Initialize standalone client cache object.
self.client_cache = None
diff --git a/autogen/agentchat/groupchat.py b/autogen/agentchat/groupchat.py
index 01c6ad709..3779683ce 100644
--- a/autogen/agentchat/groupchat.py
+++ b/autogen/agentchat/groupchat.py
@@ -9,9 +9,9 @@ from typing import Dict, List, Optional, Union, Tuple
from ..code_utils import content_str
from .agent import Agent
from .conversable_agent import ConversableAgent
+from ..runtime_logging import logging_enabled, log_new_agent
from ..graph_utils import check_graph_validity, invert_disallowed_to_allowed
-
logger = logging.getLogger(__name__)
@@ -463,6 +463,8 @@ class GroupChatManager(ConversableAgent):
system_message=system_message,
**kwargs,
)
+ if logging_enabled():
+ log_new_agent(self, locals())
# Store groupchat
self._groupchat = groupchat
diff --git a/autogen/agentchat/user_proxy_agent.py b/autogen/agentchat/user_proxy_agent.py
index 86b7e1e7b..dc68c6ec6 100644
--- a/autogen/agentchat/user_proxy_agent.py
+++ b/autogen/agentchat/user_proxy_agent.py
@@ -1,6 +1,7 @@
from typing import Callable, Dict, List, Literal, Optional, Union
from .conversable_agent import ConversableAgent
+from ..runtime_logging import logging_enabled, log_new_agent
class UserProxyAgent(ConversableAgent):
@@ -93,3 +94,6 @@ class UserProxyAgent(ConversableAgent):
if description is not None
else self.DEFAULT_USER_PROXY_AGENT_DESCRIPTIONS[human_input_mode],
)
+
+ if logging_enabled():
+ log_new_agent(self, locals())
diff --git a/autogen/logger/__init__.py b/autogen/logger/__init__.py
new file mode 100644
index 000000000..6561cab43
--- /dev/null
+++ b/autogen/logger/__init__.py
@@ -0,0 +1,4 @@
+from .logger_factory import LoggerFactory
+from .sqlite_logger import SqliteLogger
+
+__all__ = ("LoggerFactory", "SqliteLogger")
diff --git a/autogen/logger/base_logger.py b/autogen/logger/base_logger.py
new file mode 100644
index 000000000..fff78d824
--- /dev/null
+++ b/autogen/logger/base_logger.py
@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Dict, TYPE_CHECKING, Union
+import sqlite3
+import uuid
+
+from openai import OpenAI, AzureOpenAI
+from openai.types.chat import ChatCompletion
+
+if TYPE_CHECKING:
+ from autogen import ConversableAgent, OpenAIWrapper
+
+
+class BaseLogger(ABC):
+ @abstractmethod
+ def start(self) -> str:
+ """
+ Open a connection to the logging database, and start recording.
+
+ Returns:
+ session_id (str): a unique id for the logging session
+ """
+ ...
+
+ @abstractmethod
+ def log_chat_completion(
+ invocation_id: uuid.UUID,
+ client_id: int,
+ wrapper_id: int,
+ request: Dict,
+ response: Union[str, ChatCompletion],
+ is_cached: int,
+ cost: float,
+ start_time: str,
+ ) -> None:
+ """
+ Log a chat completion to database.
+
+ In AutoGen, chat completions are somewhat complicated because they are handled by the `autogen.oai.OpenAIWrapper` class.
+ One invocation to `create` can lead to multiple underlying OpenAI calls, depending on the llm_config list used, and
+ any errors or retries.
+
+ Args:
+ invocation_id (uuid): A unique identifier for the invocation to the OpenAIWrapper.create method call
+ client_id (int): A unique identifier for the underlying OpenAI client instance
+ wrapper_id (int): A unique identifier for the OpenAIWrapper instance
+ request (dict): A dictionary representing the the request or call to the OpenAI client endpoint
+ response (str or ChatCompletion): The response from OpenAI
+ is_chached (int): 1 if the response was a cache hit, 0 otherwise
+ cost(float): The cost for OpenAI response
+ start_time (str): A string representing the moment the request was initiated
+ """
+ ...
+
+ @abstractmethod
+ def log_new_agent(agent: ConversableAgent, init_args: Dict) -> None:
+ """
+ Log the birth of a new agent.
+
+ Args:
+ agent (ConversableAgent): The agent to log.
+ init_args (dict): The arguments passed to the construct the conversable agent
+ """
+ ...
+
+ @abstractmethod
+ def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict) -> None:
+ """
+ Log the birth of a new OpenAIWrapper.
+
+ Args:
+ wrapper (OpenAIWrapper): The wrapper to log.
+ init_args (dict): The arguments passed to the construct the wrapper
+ """
+ ...
+
+ @abstractmethod
+ def log_new_client(client: Union[AzureOpenAI, OpenAI], wrapper: OpenAIWrapper, init_args: Dict) -> None:
+ """
+ Log the birth of a new OpenAIWrapper.
+
+ Args:
+ wrapper (OpenAI): The OpenAI client to log.
+ init_args (dict): The arguments passed to the construct the client
+ """
+ ...
+
+ @abstractmethod
+ def stop() -> None:
+ """
+ Close the connection to the logging database, and stop logging.
+ """
+ ...
+
+ @abstractmethod
+ def get_connection() -> Union[sqlite3.Connection]:
+ """
+ Return a connection to the logging database.
+ """
+ ...
diff --git a/autogen/logger/logger_factory.py b/autogen/logger/logger_factory.py
new file mode 100644
index 000000000..282efc326
--- /dev/null
+++ b/autogen/logger/logger_factory.py
@@ -0,0 +1,17 @@
+from typing import Any, Dict, Optional
+from autogen.logger.base_logger import BaseLogger
+from autogen.logger.sqlite_logger import SqliteLogger
+
+__all__ = ("LoggerFactory",)
+
+
+class LoggerFactory:
+ @staticmethod
+ def get_logger(logger_type: str = "sqlite", config: Optional[Dict[str, Any]] = None) -> BaseLogger:
+ if config is None:
+ config = {}
+
+ if logger_type == "sqlite":
+ return SqliteLogger(config)
+ else:
+ raise ValueError(f"[logger_factory] Unknown logger type: {logger_type}")
diff --git a/autogen/logger/logger_utils.py b/autogen/logger/logger_utils.py
new file mode 100644
index 000000000..955adcf18
--- /dev/null
+++ b/autogen/logger/logger_utils.py
@@ -0,0 +1,36 @@
+import datetime
+import inspect
+from typing import Any, Dict, List, Tuple, Union
+
+__all__ = ("get_current_ts", "to_dict")
+
+
+def get_current_ts():
+ return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")
+
+
+def to_dict(
+ obj: Union[int, float, str, bool, Dict[Any, Any], List[Any], Tuple[Any, ...], Any],
+ exclude: Tuple[str] = (),
+ no_recursive: Tuple[str] = (),
+) -> Any:
+ if isinstance(obj, (int, float, str, bool)):
+ return obj
+ elif callable(obj):
+ return inspect.getsource(obj).strip()
+ elif isinstance(obj, dict):
+ return {
+ str(k): to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive)
+ for k, v in obj.items()
+ if k not in exclude
+ }
+ elif isinstance(obj, (list, tuple)):
+ return [to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive) for v in obj]
+ elif hasattr(obj, "__dict__"):
+ return {
+ str(k): to_dict(str(v)) if isinstance(v, no_recursive) else to_dict(v, exclude, no_recursive)
+ for k, v in vars(obj).items()
+ if k not in exclude
+ }
+ else:
+ return obj
diff --git a/autogen/logger/sqlite_logger.py b/autogen/logger/sqlite_logger.py
new file mode 100644
index 000000000..2fbefc169
--- /dev/null
+++ b/autogen/logger/sqlite_logger.py
@@ -0,0 +1,301 @@
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sqlite3
+import sys
+import uuid
+
+from autogen.logger.base_logger import BaseLogger
+from autogen.logger.logger_utils import get_current_ts, to_dict
+
+from openai import OpenAI, AzureOpenAI
+from openai.types.chat import ChatCompletion
+from typing import Dict, TYPE_CHECKING, Union
+
+
+if TYPE_CHECKING:
+ from autogen import ConversableAgent, OpenAIWrapper
+
+
+# this is a pointer to the module object instance itself
+this = sys.modules[__name__]
+this._session_id = None
+logger = logging.getLogger(__name__)
+
+__all__ = ("SqliteLogger",)
+
+
+class SqliteLogger(BaseLogger):
+ schema_version = 1
+
+ def __init__(self, config):
+ self.con = None
+ self.cur = None
+ self.config = config
+
+ def start(self) -> str:
+ dbname = self.config["dbname"] if "dbname" in self.config else "logs.db"
+ this._session_id = str(uuid.uuid4())
+
+ try:
+ self.con = sqlite3.connect(dbname)
+ self.cur = self.con.cursor()
+
+ query = """
+ CREATE TABLE IF NOT EXISTS chat_completions(
+ id INTEGER PRIMARY KEY,
+ invocation_id TEXT,
+ client_id INTEGER,
+ wrapper_id INTEGER,
+ session_id TEXT,
+ request TEXT,
+ response TEXT,
+ is_cached INEGER,
+ cost REAL,
+ start_time DATETIME DEFAULT CURRENT_TIMESTAMP,
+ end_time DATETIME DEFAULT CURRENT_TIMESTAMP)
+ """
+ self.cur.execute(query)
+ self.con.commit()
+
+ query = """
+ CREATE TABLE IF NOT EXISTS agents (
+ id INTEGER PRIMARY KEY, -- Key assigned by the database
+ agent_id INTEGER, -- result of python id(agent)
+ wrapper_id INTEGER, -- result of python id(agent.client)
+ session_id TEXT,
+ name TEXT, -- agent.name
+ class TEXT, -- type or class name of agent
+ init_args TEXT, -- JSON serialization of constructor
+ timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+ UNIQUE(agent_id, session_id))
+ """
+ self.cur.execute(query)
+ self.con.commit()
+
+ query = """
+ CREATE TABLE IF NOT EXISTS oai_wrappers (
+ id INTEGER PRIMARY KEY, -- Key assigned by the database
+ wrapper_id INTEGER, -- result of python id(wrapper)
+ session_id TEXT,
+ init_args TEXT, -- JSON serialization of constructor
+ timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+ UNIQUE(wrapper_id, session_id))
+ """
+ self.cur.execute(query)
+ self.con.commit()
+
+ query = """
+ CREATE TABLE IF NOT EXISTS oai_clients (
+ id INTEGER PRIMARY KEY, -- Key assigned by the database
+ client_id INTEGER, -- result of python id(client)
+ wrapper_id INTEGER, -- result of python id(wrapper)
+ session_id TEXT,
+ class TEXT, -- type or class name of client
+ init_args TEXT, -- JSON serialization of constructor
+ timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
+ UNIQUE(client_id, session_id))
+ """
+ self.cur.execute(query)
+ self.con.commit()
+
+ query = """
+ CREATE TABLE IF NOT EXISTS version (
+ id INTEGER PRIMARY KEY CHECK (id = 1), -- id of the logging database
+ version_number INTEGER NOT NULL -- version of the logging database
+ );
+ """
+ self.cur.execute(query)
+ self.con.commit()
+
+ current_verion = self._get_current_db_version()
+ if current_verion is None:
+ self.cur.execute(
+ "INSERT INTO version (id, version_number) VALUES (1, ?);", (SqliteLogger.schema_version,)
+ )
+ self.con.commit()
+
+ self._apply_migration(dbname)
+
+ except sqlite3.Error as e:
+ logger.error(f"[SqliteLogger] start logging error: {e}")
+ finally:
+ return this._session_id
+
+ def _get_current_db_version(self):
+ self.cur.execute("SELECT version_number FROM version ORDER BY id DESC LIMIT 1")
+ result = self.cur.fetchone()
+ return result[0] if result else None
+
+ # Example migration script name format: 002_update_agents_table.sql
+ def _apply_migration(self, dbname, migrations_dir="./migrations"):
+ current_version = self._get_current_db_version()
+ if os.path.isdir(migrations_dir):
+ migrations = sorted(os.listdir(migrations_dir))
+ else:
+ logger.info("no migration scripts, skip...")
+ return
+
+ migrations_to_apply = [m for m in migrations if int(m.split("_")[0]) > current_version]
+
+ for script in migrations_to_apply:
+ with open(script, "r") as f:
+ migration_sql = f.read()
+ self.con.executescript(migration_sql)
+ self.con.commit()
+
+ latest_version = int(script.split("_")[0])
+ self.cur.execute("UPDATE version SET version_number = ? WHERE id = 1", (latest_version))
+ self.con.commit()
+
+ def log_chat_completion(
+ self,
+ invocation_id: uuid.UUID,
+ client_id: int,
+ wrapper_id: int,
+ request: Dict,
+ response: Union[str, ChatCompletion],
+ is_cached: int,
+ cost: float,
+ start_time: str,
+ ) -> None:
+ if self.con is None:
+ return
+
+ end_time = get_current_ts()
+
+ if response is None or isinstance(response, str):
+ response_messages = json.dumps({"response": response})
+ else:
+ response_messages = json.dumps(to_dict(response), indent=4)
+
+ query = """
+ INSERT INTO chat_completions (
+ invocation_id, client_id, wrapper_id, session_id, request, response, is_cached, cost, start_time, end_time
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ """
+
+ try:
+ self.cur.execute(
+ query,
+ (
+ invocation_id,
+ client_id,
+ wrapper_id,
+ this._session_id,
+ json.dumps(request),
+ response_messages,
+ is_cached,
+ cost,
+ start_time,
+ end_time,
+ ),
+ )
+ self.con.commit()
+ except sqlite3.Error as e:
+ logger.error(f"[SqliteLogger] log_chat_completion error: {e}")
+
+ def log_new_agent(self, agent: ConversableAgent, init_args: Dict) -> None:
+ from autogen import Agent
+
+ if self.con is None:
+ return
+
+ args = to_dict(
+ init_args,
+ exclude=("self", "__class__", "api_key", "organization", "base_url", "azure_endpoint"),
+ no_recursive=(Agent),
+ )
+
+ # We do an upsert since both the superclass and subclass may call this method (in that order)
+ query = """
+ INSERT INTO agents (agent_id, wrapper_id, session_id, name, class, init_args, timestamp) VALUES (?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT (agent_id, session_id) DO UPDATE SET
+ wrapper_id = excluded.wrapper_id,
+ name = excluded.name,
+ class = excluded.class,
+ init_args = excluded.init_args,
+ timestamp = excluded.timestamp
+ """
+ try:
+ self.cur.execute(
+ query,
+ (
+ id(agent),
+ agent.client.wrapper_id if hasattr(agent, "client") and agent.client is not None else "",
+ this._session_id,
+ agent.name if hasattr(agent, "name") and agent.name is not None else "",
+ type(agent).__name__,
+ json.dumps(args),
+ get_current_ts(),
+ ),
+ )
+ self.con.commit()
+ except sqlite3.Error as e:
+ logger.error(f"[SqliteLogger] log_new_agent error: {e}")
+
+ def log_new_wrapper(self, wrapper: OpenAIWrapper, init_args: Dict) -> None:
+ if self.con is None:
+ return
+
+ args = to_dict(
+ init_args, exclude=("self", "__class__", "api_key", "organization", "base_url", "azure_endpoint")
+ )
+
+ query = """
+ INSERT INTO oai_wrappers (wrapper_id, session_id, init_args, timestamp) VALUES (?, ?, ?, ?)
+ ON CONFLICT (wrapper_id, session_id) DO NOTHING;
+ """
+ try:
+ self.cur.execute(
+ query,
+ (
+ id(wrapper),
+ this._session_id,
+ json.dumps(args),
+ get_current_ts(),
+ ),
+ )
+ self.con.commit()
+ except sqlite3.Error as e:
+ logger.error(f"[SqliteLogger] log_new_wrapper error: {e}")
+
+ def log_new_client(self, client: Union[AzureOpenAI, OpenAI], wrapper: OpenAIWrapper, init_args: Dict) -> None:
+ if self.con is None:
+ return
+
+ args = to_dict(
+ init_args, exclude=("self", "__class__", "api_key", "organization", "base_url", "azure_endpoint")
+ )
+
+ query = """
+ INSERT INTO oai_clients (client_id, wrapper_id, session_id, class, init_args, timestamp) VALUES (?, ?, ?, ?, ?, ?)
+ ON CONFLICT (client_id, session_id) DO NOTHING;
+ """
+ try:
+ self.cur.execute(
+ query,
+ (
+ id(client),
+ id(wrapper),
+ this._session_id,
+ type(client).__name__,
+ json.dumps(args),
+ get_current_ts(),
+ ),
+ )
+ self.con.commit()
+ except sqlite3.Error as e:
+ logger.error(f"[SqliteLogger] log_new_client error: {e}")
+
+ def stop(self) -> None:
+ if self.con:
+ self.con.close()
+ self.con = None
+ self.cur = None
+
+ def get_connection(self) -> sqlite3.Connection:
+ if self.con:
+ return self.con
diff --git a/autogen/oai/client.py b/autogen/oai/client.py
index 19b937c0e..581c30698 100644
--- a/autogen/oai/client.py
+++ b/autogen/oai/client.py
@@ -4,6 +4,7 @@ import sys
from typing import Any, List, Optional, Dict, Callable, Tuple, Union
import logging
import inspect
+import uuid
from flaml.automl.logger import logger_formatter
from pydantic import BaseModel
@@ -13,6 +14,9 @@ from autogen.cache.cache import Cache
from autogen.oai.openai_utils import get_key, is_valid_api_key, OAI_PRICE1K
from autogen.token_count_utils import count_token
+from autogen.runtime_logging import logging_enabled, log_chat_completion, log_new_client, log_new_wrapper
+from autogen.logger.logger_utils import get_current_ts
+
TOOL_ENABLED = False
try:
import openai
@@ -353,6 +357,9 @@ class OpenAIWrapper:
base_config: base config. It can contain both keyword arguments for openai client
and additional kwargs.
"""
+
+ if logging_enabled():
+ log_new_wrapper(self, locals())
openai_config, extra_kwargs = self._separate_openai_config(base_config)
if type(config_list) is list and len(config_list) == 0:
logger.warning("openai client was provided with an empty config_list, which may not be intended.")
@@ -370,6 +377,7 @@ class OpenAIWrapper:
else:
self._register_default_client(extra_kwargs, openai_config)
self._config_list = [extra_kwargs]
+ self.wrapper_id = id(self)
def _separate_openai_config(self, config: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]:
"""Separate the config into openai_config and extra_kwargs."""
@@ -408,12 +416,18 @@ class OpenAIWrapper:
logger.info(
f"Detected custom model client in config: {model_client_cls_name}, model client can not be used until register_model_client is called."
)
+ # TODO: logging for custom client
else:
if api_type is not None and api_type.startswith("azure"):
self._configure_azure_openai(config, openai_config)
- self._clients.append(OpenAIClient(AzureOpenAI(**openai_config)))
+ client = AzureOpenAI(**openai_config)
+ self._clients.append(OpenAIClient(client))
else:
- self._clients.append(OpenAIClient(OpenAI(**openai_config)))
+ client = OpenAI(**openai_config)
+ self._clients.append(OpenAIClient(client))
+
+ if logging_enabled():
+ log_new_client(client, self, openai_config)
def register_model_client(self, model_client_cls: ModelClient, **kwargs):
"""Register a model client.
@@ -527,6 +541,7 @@ class OpenAIWrapper:
"""
if ERROR:
raise ERROR
+ invocation_id = str(uuid.uuid4())
last = len(self._clients) - 1
# Check if all configs in config list are activated
non_activated = [
@@ -567,6 +582,8 @@ class OpenAIWrapper:
with cache_client as cache:
# Try to get the response from cache
key = get_key(params)
+ request_ts = get_current_ts()
+
response: ModelClient.ModelClientResponseProtocol = cache.get(key, None)
if response is not None:
@@ -578,6 +595,20 @@ class OpenAIWrapper:
response.cost = client.cost(response)
cache.set(key, response)
total_usage = client.get_usage(response)
+
+ if logging_enabled():
+ # Log the cache hit
+ log_chat_completion(
+ invocation_id=invocation_id,
+ client_id=id(client),
+ wrapper_id=id(self),
+ request=params,
+ response=response,
+ is_cached=1,
+ cost=response.cost,
+ start_time=request_ts,
+ )
+
# check the filter
pass_filter = filter_func is None or filter_func(context=context, response=response)
if pass_filter or i == last:
@@ -588,6 +619,7 @@ class OpenAIWrapper:
return response
continue # filter is not passed; try the next config
try:
+ request_ts = get_current_ts()
response = client.create(params)
except APITimeoutError as err:
logger.debug(f"config {i} timed out", exc_info=True)
@@ -597,6 +629,18 @@ class OpenAIWrapper:
) from err
except APIError as err:
error_code = getattr(err, "code", None)
+ if logging_enabled():
+ log_chat_completion(
+ invocation_id=invocation_id,
+ client_id=id(client),
+ wrapper_id=id(self),
+ request=params,
+ response=f"error_code:{error_code}, config {i} failed",
+ is_cached=0,
+ cost=0,
+ start_time=request_ts,
+ )
+
if error_code == "content_filter":
# raise the error for content_filter
raise
@@ -614,6 +658,18 @@ class OpenAIWrapper:
with cache_client as cache:
cache.set(key, response)
+ if logging_enabled():
+ log_chat_completion(
+ invocation_id=invocation_id,
+ client_id=id(client),
+ wrapper_id=id(self),
+ request=params,
+ response=response,
+ is_cached=0,
+ cost=response.cost,
+ start_time=request_ts,
+ )
+
response.message_retrieval_function = client.message_retrieval
# check the filter
pass_filter = filter_func is None or filter_func(context=context, response=response)
@@ -836,6 +892,3 @@ class OpenAIWrapper:
A list of text, or a list of ChatCompletion objects if function_call/tool_calls are present.
"""
return response.message_retrieval_function(response)
-
-
-# TODO: logging
diff --git a/autogen/runtime_logging.py b/autogen/runtime_logging.py
new file mode 100644
index 000000000..6f17810c9
--- /dev/null
+++ b/autogen/runtime_logging.py
@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from autogen.logger.logger_factory import LoggerFactory
+import sqlite3
+from typing import Any, Dict, Optional, TYPE_CHECKING, Union
+import uuid
+
+from openai import OpenAI, AzureOpenAI
+from openai.types.chat import ChatCompletion
+
+if TYPE_CHECKING:
+ from autogen import ConversableAgent, OpenAIWrapper
+
+autogen_logger = None
+is_logging = False
+
+
+def start(logger_type: str = "sqlite", config: Optional[Dict[str, Any]] = None) -> str:
+ global autogen_logger
+ global is_logging
+
+ if autogen_logger is None:
+ autogen_logger = LoggerFactory.get_logger(logger_type=logger_type, config=config)
+
+ session_id = autogen_logger.start()
+ is_logging = True
+
+ return session_id
+
+
+def log_chat_completion(
+ invocation_id: uuid.UUID,
+ client_id: int,
+ wrapper_id: int,
+ request: Dict,
+ response: Union[str, ChatCompletion],
+ is_cached: int,
+ cost: float,
+ start_time: str,
+) -> None:
+ autogen_logger.log_chat_completion(
+ invocation_id, client_id, wrapper_id, request, response, is_cached, cost, start_time
+ )
+
+
+def log_new_agent(agent: ConversableAgent, init_args: Dict) -> None:
+ autogen_logger.log_new_agent(agent, init_args)
+
+
+def log_new_wrapper(wrapper: OpenAIWrapper, init_args: Dict) -> None:
+ autogen_logger.log_new_wrapper(wrapper, init_args)
+
+
+def log_new_client(client: Union[AzureOpenAI, OpenAI], wrapper: OpenAIWrapper, init_args: Dict) -> None:
+ autogen_logger.log_new_client(client, wrapper, init_args)
+
+
+def stop() -> None:
+ global is_logging
+ if autogen_logger:
+ autogen_logger.stop()
+ is_logging = False
+
+
+def get_connection() -> Union[sqlite3.Connection]:
+ return autogen_logger.get_connection()
+
+
+def logging_enabled() -> bool:
+ return is_logging
diff --git a/notebook/agentchat_logging.ipynb b/notebook/agentchat_logging.ipynb
new file mode 100644
index 000000000..5d8a824e9
--- /dev/null
+++ b/notebook/agentchat_logging.ipynb
@@ -0,0 +1,317 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<!--\n",
+ "tags: [\"logging\", \"debugging\"]\n",
+ "description: |\n",
+ " Provide capabilities of runtime logging for debugging and performance analysis.\n",
+ "-->\n",
+ "\n",
+ "# Runtime Logging with AutoGen \n",
+ "\n",
+ "AutoGen offers utilities to log data for debugging and performance analysis. This notebook demonstrates how to use them. \n",
+ "\n",
+ "In general, users can initiate logging by calling `autogen.runtime_logging.start()` and stop logging by calling `autogen.runtime_logging.stop()`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Logging session ID: 6e08f3e0-392b-434e-8b69-4ab36c4fcf99\n",
+ "\u001b[33muser_proxy\u001b[0m (to assistant):\n",
+ "\n",
+ "What is the height of the Eiffel Tower? Only respond with the answer and terminate\n",
+ "\n",
+ "--------------------------------------------------------------------------------\n",
+ "\u001b[33massistant\u001b[0m (to user_proxy):\n",
+ "\n",
+ "The height of the Eiffel Tower is approximately 330 meters.\n",
+ "\n",
+ "TERMINATE\n",
+ "\n",
+ "--------------------------------------------------------------------------------\n"
+ ]
+ }
+ ],
+ "source": [
+ "import json\n",
+ "import autogen\n",
+ "from autogen import AssistantAgent, UserProxyAgent\n",
+ "import pandas as pd\n",
+ "\n",
+ "# Setup API key. Add your own API key to config file or environment variable\n",
+ "llm_config = {\n",
+ " \"config_list\": autogen.config_list_from_json(\n",
+ " env_or_file=\"OAI_CONFIG_LIST\",\n",
+ " ),\n",
+ " \"temperature\": 0.9,\n",
+ "}\n",
+ "\n",
+ "# Start logging\n",
+ "logging_session_id = autogen.runtime_logging.start(config={\"dbname\": \"logs.db\"})\n",
+ "print(\"Logging session ID: \" + str(logging_session_id))\n",
+ "\n",
+ "# Create an agent workflow and run it\n",
+ "assistant = AssistantAgent(name=\"assistant\", llm_config=llm_config)\n",
+ "user_proxy = UserProxyAgent(\n",
+ " name=\"user_proxy\",\n",
+ " code_execution_config=False,\n",
+ " human_input_mode=\"NEVER\",\n",
+ " is_termination_msg=lambda msg: \"TERMINATE\" in msg[\"content\"],\n",
+ ")\n",
+ "\n",
+ "user_proxy.initiate_chat(\n",
+ " assistant, message=\"What is the height of the Eiffel Tower? Only respond with the answer and terminate\"\n",
+ ")\n",
+ "autogen.runtime_logging.stop()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Getting Data from the SQLite Database \n",
+ "\n",
+ "`logs.db` should be generated, by default it's using SQLite database. You can view the data with GUI tool like `sqlitebrowser`, using SQLite command line shell or using python script:\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_log(dbname=\"logs.db\", table=\"chat_completions\"):\n",
+ " import sqlite3\n",
+ "\n",
+ " con = sqlite3.connect(dbname)\n",
+ " query = f\"SELECT * from {table}\"\n",
+ " cursor = con.execute(query)\n",
+ " rows = cursor.fetchall()\n",
+ " column_names = [description[0] for description in cursor.description]\n",
+ " data = [dict(zip(column_names, row)) for row in rows]\n",
+ " con.close()\n",
+ " return data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<div>\n",
+ "<style scoped>\n",
+ " .dataframe tbody tr th:only-of-type {\n",
+ " vertical-align: middle;\n",
+ " }\n",
+ "\n",
+ " .dataframe tbody tr th {\n",
+ " vertical-align: top;\n",
+ " }\n",
+ "\n",
+ " .dataframe thead th {\n",
+ " text-align: right;\n",
+ " }\n",
+ "</style>\n",
+ "<table border=\"1\" class=\"dataframe\">\n",
+ " <thead>\n",
+ " <tr style=\"text-align: right;\">\n",
+ " <th></th>\n",
+ " <th>id</th>\n",
+ " <th>invocation_id</th>\n",
+ " <th>client_id</th>\n",
+ " <th>wrapper_id</th>\n",
+ " <th>session_id</th>\n",
+ " <th>request</th>\n",
+ " <th>response</th>\n",
+ " <th>is_cached</th>\n",
+ " <th>cost</th>\n",
+ " <th>start_time</th>\n",
+ " <th>end_time</th>\n",
+ " <th>total_tokens</th>\n",
+ " </tr>\n",
+ " </thead>\n",
+ " <tbody>\n",
+ " <tr>\n",
+ " <th>0</th>\n",
+ " <td>1</td>\n",
+ " <td>e8bb00d7-6da5-4407-a949-e19b55d53da8</td>\n",
+ " <td>139819167322784</td>\n",
+ " <td>139823225568704</td>\n",
+ " <td>8821a150-8c78-4d05-a858-8a64f1d18648</td>\n",
+ " <td>You are a helpful AI assistant.\\nSolve tasks u...</td>\n",
+ " <td>The height of the Eiffel Tower is approximatel...</td>\n",
+ " <td>1</td>\n",
+ " <td>0.01572</td>\n",
+ " <td>2024-02-13 15:06:22.082896</td>\n",
+ " <td>2024-02-13 15:06:22.083169</td>\n",
+ " <td>507</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>1</th>\n",
+ " <td>2</td>\n",
+ " <td>c8522790-0067-484b-bb37-d39ae80db98b</td>\n",
+ " <td>139823225568656</td>\n",
+ " <td>139823225563040</td>\n",
+ " <td>fb0ef547-a2ac-428b-8c20-a5e63263b8e1</td>\n",
+ " <td>You are a helpful AI assistant.\\nSolve tasks u...</td>\n",
+ " <td>The height of the Eiffel Tower is approximatel...</td>\n",
+ " <td>1</td>\n",
+ " <td>0.01572</td>\n",
+ " <td>2024-02-13 15:06:23.498758</td>\n",
+ " <td>2024-02-13 15:06:23.499045</td>\n",
+ " <td>507</td>\n",
+ " </tr>\n",
+ " <tr>\n",
+ " <th>2</th>\n",
+ " <td>3</td>\n",
+ " <td>91c3f6c0-c6f7-4306-89cd-f304c9556de4</td>\n",
+ " <td>139823225449024</td>\n",
+ " <td>139819166072448</td>\n",
+ " <td>6e08f3e0-392b-434e-8b69-4ab36c4fcf99</td>\n",
+ " <td>You are a helpful AI assistant.\\nSolve tasks u...</td>\n",
+ " <td>The height of the Eiffel Tower is approximatel...</td>\n",
+ " <td>1</td>\n",
+ " <td>0.01572</td>\n",
+ " <td>2024-02-13 15:06:24.688990</td>\n",
+ " <td>2024-02-13 15:06:24.689238</td>\n",
+ " <td>507</td>\n",
+ " </tr>\n",
+ " </tbody>\n",
+ "</table>\n",
+ "</div>"
+ ],
+ "text/plain": [
+ " id invocation_id client_id wrapper_id \\\n",
+ "0 1 e8bb00d7-6da5-4407-a949-e19b55d53da8 139819167322784 139823225568704 \n",
+ "1 2 c8522790-0067-484b-bb37-d39ae80db98b 139823225568656 139823225563040 \n",
+ "2 3 91c3f6c0-c6f7-4306-89cd-f304c9556de4 139823225449024 139819166072448 \n",
+ "\n",
+ " session_id \\\n",
+ "0 8821a150-8c78-4d05-a858-8a64f1d18648 \n",
+ "1 fb0ef547-a2ac-428b-8c20-a5e63263b8e1 \n",
+ "2 6e08f3e0-392b-434e-8b69-4ab36c4fcf99 \n",
+ "\n",
+ " request \\\n",
+ "0 You are a helpful AI assistant.\\nSolve tasks u... \n",
+ "1 You are a helpful AI assistant.\\nSolve tasks u... \n",
+ "2 You are a helpful AI assistant.\\nSolve tasks u... \n",
+ "\n",
+ " response is_cached cost \\\n",
+ "0 The height of the Eiffel Tower is approximatel... 1 0.01572 \n",
+ "1 The height of the Eiffel Tower is approximatel... 1 0.01572 \n",
+ "2 The height of the Eiffel Tower is approximatel... 1 0.01572 \n",
+ "\n",
+ " start_time end_time total_tokens \n",
+ "0 2024-02-13 15:06:22.082896 2024-02-13 15:06:22.083169 507 \n",
+ "1 2024-02-13 15:06:23.498758 2024-02-13 15:06:23.499045 507 \n",
+ "2 2024-02-13 15:06:24.688990 2024-02-13 15:06:24.689238 507 "
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "def str_to_dict(s):\n",
+ " return json.loads(s)\n",
+ "\n",
+ "\n",
+ "log_data = get_log()\n",
+ "log_data_df = pd.DataFrame(log_data)\n",
+ "\n",
+ "log_data_df[\"total_tokens\"] = log_data_df.apply(\n",
+ " lambda row: str_to_dict(row[\"response\"])[\"usage\"][\"total_tokens\"], axis=1\n",
+ ")\n",
+ "\n",
+ "log_data_df[\"request\"] = log_data_df.apply(lambda row: str_to_dict(row[\"request\"])[\"messages\"][0][\"content\"], axis=1)\n",
+ "\n",
+ "log_data_df[\"response\"] = log_data_df.apply(\n",
+ " lambda row: str_to_dict(row[\"response\"])[\"choices\"][0][\"message\"][\"content\"], axis=1\n",
+ ")\n",
+ "\n",
+ "log_data_df"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Computing Cost \n",
+ "\n",
+ "One use case of logging data is to compute the cost of a session."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Total tokens for all sessions: 1521, total cost: 0.0472\n",
+ "Total tokens for session 6e08f3e0-392b-434e-8b69-4ab36c4fcf99: 507, cost: 0.0157\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Sum totoal tokens for all sessions\n",
+ "total_tokens = log_data_df[\"total_tokens\"].sum()\n",
+ "\n",
+ "# Sum total cost for all sessions\n",
+ "total_cost = log_data_df[\"cost\"].sum()\n",
+ "\n",
+ "# Total tokens for specific session\n",
+ "session_tokens = log_data_df[log_data_df[\"session_id\"] == logging_session_id][\"total_tokens\"].sum()\n",
+ "session_cost = log_data_df[log_data_df[\"session_id\"] == logging_session_id][\"cost\"].sum()\n",
+ "\n",
+ "print(\"Total tokens for all sessions: \" + str(total_tokens) + \", total cost: \" + str(round(total_cost, 4)))\n",
+ "print(\n",
+ " \"Total tokens for session \"\n",
+ " + str(logging_session_id)\n",
+ " + \": \"\n",
+ " + str(session_tokens)\n",
+ " + \", cost: \"\n",
+ " + str(round(session_cost, 4))\n",
+ ")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "autog",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/test/agentchat/test_agent_logging.py b/test/agentchat/test_agent_logging.py
new file mode 100644
index 000000000..3c99867b5
--- /dev/null
+++ b/test/agentchat/test_agent_logging.py
@@ -0,0 +1,238 @@
+import pytest
+import autogen
+import autogen.runtime_logging
+import json
+import sys
+import uuid
+import sqlite3
+
+from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST
+from conftest import skip_openai
+
+
+TEACHER_MESSAGE = """
+ You are roleplaying a math teacher, and your job is to help your students with linear algebra.
+ Keep your explanations short.
+"""
+
+STUDENT_MESSAGE = """
+ You are roleplaying a high school student strugling with linear algebra.
+ Regardless how well the teacher explains things to you, you just don't quite get it.
+ Keep your questions short.
+"""
+
+CHAT_COMPLETIONS_QUERY = """SELECT id, invocation_id, client_id, wrapper_id, session_id,
+ request, response, is_cached, cost, start_time, end_time FROM chat_completions;"""
+
+AGENTS_QUERY = "SELECT id, agent_id, wrapper_id, session_id, name, class, init_args, timestamp FROM agents"
+
+OAI_CLIENTS_QUERY = "SELECT id, client_id, wrapper_id, session_id, class, init_args, timestamp FROM oai_clients"
+
+OAI_WRAPPERS_QUERY = "SELECT id, wrapper_id, session_id, init_args, timestamp FROM oai_wrappers"
+
+
+if not skip_openai:
+ config_list = autogen.config_list_from_json(
+ OAI_CONFIG_LIST,
+ filter_dict={
+ "model": ["gpt-4", "gpt-4-0314", "gpt-4-32k", "gpt-4-32k-0314", "gpt-4-32k-v0314"],
+ },
+ file_location=KEY_LOC,
+ )
+
+ llm_config = {"config_list": config_list}
+
+ num_of_configs = len(config_list)
+###############################################################
+
+
[email protected](scope="function")
+def db_connection():
+ autogen.runtime_logging.start(config={"dbname": ":memory:"})
+ con = autogen.runtime_logging.get_connection()
+ con.row_factory = sqlite3.Row
+ yield con
+
+ autogen.runtime_logging.stop()
+
+
+ sys.platform in ["darwin", "win32"] or skip_openai,
+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
+)
+def test_two_agents_logging(db_connection):
+ cur = db_connection.cursor()
+
+ teacher = autogen.AssistantAgent(
+ "teacher",
+ system_message=TEACHER_MESSAGE,
+ is_termination_msg=lambda x: x.get("content", "").find("TERMINATE") >= 0,
+ llm_config=llm_config,
+ max_consecutive_auto_reply=2,
+ )
+
+ student = autogen.AssistantAgent(
+ "student",
+ system_message=STUDENT_MESSAGE,
+ is_termination_msg=lambda x: x.get("content", "").find("TERMINATE") >= 0,
+ llm_config=llm_config,
+ max_consecutive_auto_reply=1,
+ )
+
+ student.initiate_chat(
+ teacher,
+ message="Can you explain the difference between eigenvalues and singular values again?",
+ )
+
+ # Verify log completions table
+ cur.execute(CHAT_COMPLETIONS_QUERY)
+ rows = cur.fetchall()
+
+ assert len(rows) == 3
+ session_id = rows[0]["session_id"]
+
+ for idx, row in enumerate(rows):
+ assert (
+ row["invocation_id"] and str(uuid.UUID(row["invocation_id"], version=4)) == row["invocation_id"]
+ ), "invocation id is not valid uuid"
+ assert row["client_id"], "client id is empty"
+ assert row["wrapper_id"], "wrapper id is empty"
+ assert row["session_id"] and row["session_id"] == session_id
+
+ request = json.loads(row["request"])
+ first_request_message = request["messages"][0]["content"]
+ first_request_role = request["messages"][0]["role"]
+
+ if idx == 0 or idx == 2:
+ assert first_request_message == TEACHER_MESSAGE
+ elif idx == 1:
+ assert first_request_message == STUDENT_MESSAGE
+ assert first_request_role == "system"
+
+ response = json.loads(row["response"])
+ assert "choices" in response and len(response["choices"]) > 0
+
+ assert row["cost"] > 0
+ assert row["start_time"], "start timestamp is empty"
+ assert row["end_time"], "end timestamp is empty"
+
+ # Verify agents table
+ cur.execute(AGENTS_QUERY)
+ rows = cur.fetchall()
+
+ assert len(rows) == 2
+
+ session_id = rows[0]["session_id"]
+ for idx, row in enumerate(rows):
+ assert row["wrapper_id"], "wrapper id is empty"
+ assert row["session_id"] and row["session_id"] == session_id
+
+ agent = json.loads(row["init_args"])
+ if idx == 0:
+ assert row["name"] == "teacher"
+ assert agent["name"] == "teacher"
+ agent["system_message"] == TEACHER_MESSAGE
+ elif idx == 1:
+ assert row["name"] == "student"
+ assert agent["name"] == "student"
+ agent["system_message"] = STUDENT_MESSAGE
+
+ assert "api_key" not in row["init_args"]
+ assert row["timestamp"], "timestamp is empty"
+
+ # Verify oai client table
+ cur.execute(OAI_CLIENTS_QUERY)
+ rows = cur.fetchall()
+
+ assert len(rows) == num_of_configs * 2 # two agents
+
+ session_id = rows[0]["session_id"]
+ for row in rows:
+ assert row["client_id"], "client id is empty"
+ assert row["wrapper_id"], "wrapper id is empty"
+ assert row["session_id"] and row["session_id"] == session_id
+ assert row["class"] in ["AzureOpenAI", "OpenAI"]
+ init_args = json.loads(row["init_args"])
+ assert "api_version" in init_args
+ assert row["timestamp"], "timestamp is empty"
+
+ # Verify oai wrapper table
+ cur.execute(OAI_WRAPPERS_QUERY)
+ rows = cur.fetchall()
+
+ session_id = rows[0]["session_id"]
+
+ for row in rows:
+ assert row["wrapper_id"], "wrapper id is empty"
+ assert row["session_id"] and row["session_id"] == session_id
+ init_args = json.loads(row["init_args"])
+ assert "config_list" in init_args
+ assert len(init_args["config_list"]) > 0
+ assert row["timestamp"], "timestamp is empty"
+
+
+ sys.platform in ["darwin", "win32"] or skip_openai,
+ reason="do not run on MacOS or windows OR dependency is not installed OR requested to skip",
+)
+def test_groupchat_logging(db_connection):
+ cur = db_connection.cursor()
+
+ teacher = autogen.AssistantAgent(
+ "teacher",
+ system_message=TEACHER_MESSAGE,
+ is_termination_msg=lambda x: x.get("content", "").find("TERMINATE") >= 0,
+ llm_config=llm_config,
+ max_consecutive_auto_reply=2,
+ )
+
+ student = autogen.AssistantAgent(
+ "student",
+ system_message=STUDENT_MESSAGE,
+ is_termination_msg=lambda x: x.get("content", "").find("TERMINATE") >= 0,
+ llm_config=llm_config,
+ max_consecutive_auto_reply=1,
+ )
+
+ groupchat = autogen.GroupChat(
+ agents=[teacher, student], messages=[], max_round=3, speaker_selection_method="round_robin"
+ )
+
+ group_chat_manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config)
+
+ student.initiate_chat(
+ group_chat_manager,
+ message="Can you explain the difference between eigenvalues and singular values again?",
+ )
+
+ # Verify chat_completions message
+ cur.execute(CHAT_COMPLETIONS_QUERY)
+ rows = cur.fetchall()
+ assert len(rows) == 2 # max_round - 1
+
+ # Verify group chat manager agent
+ cur.execute(AGENTS_QUERY)
+ rows = cur.fetchall()
+ assert len(rows) == 3
+
+ chat_manager_query = "SELECT agent_id, name, class, init_args FROM agents WHERE name = 'chat_manager'"
+ cur.execute(chat_manager_query)
+ rows = cur.fetchall()
+ assert len(rows) == 1
+
+ # Verify oai clients
+ cur.execute(OAI_CLIENTS_QUERY)
+ rows = cur.fetchall()
+ assert len(rows) == num_of_configs * 3 # three agents
+
+ # Verify oai wrappers
+ cur.execute(OAI_WRAPPERS_QUERY)
+ rows = cur.fetchall()
+ assert len(rows) == 3
+
+ # Verify schema version
+ version_query = "SELECT id, version_number from version"
+ cur.execute(version_query)
+ rows = cur.fetchall()
+ assert len(rows) == 1
+ assert rows[0]["id"] == 1 and rows[0]["version_number"] == 1
diff --git a/test/test_logging.py b/test/test_logging.py
new file mode 100644
index 000000000..694b9c1d5
--- /dev/null
+++ b/test/test_logging.py
@@ -0,0 +1,268 @@
+import autogen.runtime_logging
+import uuid
+import json
+import pytest
+import sqlite3
+
+from autogen.logger.logger_utils import get_current_ts, to_dict
+from openai import AzureOpenAI
+from unittest.mock import patch, Mock
+
+
+SAMPLE_CHAT_REQUEST = json.loads(
+ """
+{
+ "messages": [
+ {
+ "content": "You are roleplaying a high school student strugling with linear algebra. Regardless how well the teacher explains things to you, you just don't quite get it. Keep your questions short.",
+ "role": "system"
+ },
+ {
+ "content": "Can you explain the difference between eigenvalues and singular values again?",
+ "role": "assistant"
+ },
+ {
+ "content": "Certainly!\\n\\nEigenvalues are associated with square matrices. They are the scalars, \\u03bb, that satisfy the equation\\n\\nA*x = \\u03bb*x\\n\\nwhere A is a square matrix, x is a nonzero vector (the eigenvector), and \\u03bb is the eigenvalue. The eigenvalue equation shows how the vector x is stretched or shrunk by the matrix A.\\n\\nSingular values, on the other hand, are associated with any m x n matrix, whether square or rectangular. They come from the matrix's singular value decomposition (SVD) and are the square roots of the non-negative eigenvalues of the matrix A*A^T or A^T*A (where A^T is the transpose of A). Singular values, denoted often by \\u03c3, represent the magnitude of the principal axes of the data's distribution and are always non-negative.\\n\\nTo sum up, eigenvalues relate to how a matrix scales vectors (specific to square matrices), while singular values give a measure of how a matrix stretches space (applicable to all matrices).",
+ "role": "user"
+ }
+ ],
+ "model": "gpt-4"
+}
+"""
+)
+
+SAMPLE_CHAT_RESPONSE = json.loads(
+ """
+{
+ "id": "chatcmpl-8k57oSg1fz2JwpMcEOWMqUvwjf0cb",
+ "choices": [
+ {
+ "finish_reason": "stop",
+ "index": 0,
+ "logprobs": null,
+ "message": {
+ "content": "Oh, wait, I don't think I completely understand the concept of matrix multiplication. Could you break down how you multiply two matrices together?",
+ "role": "assistant",
+ "function_call": null,
+ "tool_calls": null
+ }
+ }
+ ],
+ "created": 1705993480,
+ "model": "gpt-4",
+ "object": "chat.completion",
+ "system_fingerprint": "fp_6d044fb900",
+ "usage": {
+ "completion_tokens": 28,
+ "prompt_tokens": 274,
+ "total_tokens": 302
+ }
+}
+"""
+)
+
+###############################################################
+
+
[email protected](scope="function")
+def db_connection():
+ autogen.runtime_logging.start(config={"dbname": ":memory:"})
+ con = autogen.runtime_logging.get_connection()
+ con.row_factory = sqlite3.Row
+ yield con
+
+ autogen.runtime_logging.stop()
+
+
+def get_sample_chat_completion(response):
+ return {
+ "invocation_id": str(uuid.uuid4()),
+ "client_id": 140609438577184,
+ "wrapper_id": 140610167717744,
+ "request": SAMPLE_CHAT_REQUEST,
+ "response": response,
+ "is_cached": 0,
+ "cost": 0.347,
+ "start_time": get_current_ts(),
+ }
+
+
+ "response, expected_logged_response",
+ [
+ (SAMPLE_CHAT_RESPONSE, SAMPLE_CHAT_RESPONSE),
+ (None, {"response": None}),
+ ("error in response", {"response": "error in response"}),
+ ],
+)
+def test_log_completion(response, expected_logged_response, db_connection):
+ cur = db_connection.cursor()
+
+ sample_completion = get_sample_chat_completion(response)
+ autogen.runtime_logging.log_chat_completion(**sample_completion)
+
+ query = """
+ SELECT invocation_id, client_id, wrapper_id, request, response, is_cached,
+ cost, start_time FROM chat_completions
+ """
+
+ for row in cur.execute(query):
+ assert row["invocation_id"] == sample_completion["invocation_id"]
+ assert row["client_id"] == sample_completion["client_id"]
+ assert row["wrapper_id"] == sample_completion["wrapper_id"]
+ assert json.loads(row["request"]) == sample_completion["request"]
+ assert json.loads(row["response"]) == expected_logged_response
+ assert row["is_cached"] == sample_completion["is_cached"]
+ assert row["cost"] == sample_completion["cost"]
+ assert row["start_time"] == sample_completion["start_time"]
+
+
+def test_log_new_agent(db_connection):
+ from autogen import AssistantAgent
+
+ cur = db_connection.cursor()
+ agent_name = "some_assistant"
+ config_list = [{"model": "gpt-4", "api_key": "some_key"}]
+
+ agent = AssistantAgent(agent_name, llm_config={"config_list": config_list})
+ init_args = {"foo": "bar", "baz": {"other_key": "other_val"}, "a": None}
+
+ autogen.runtime_logging.log_new_agent(agent, init_args)
+
+ query = """
+ SELECT session_id, name, class, init_args FROM agents
+ """
+
+ for row in cur.execute(query):
+ assert (
+ row["session_id"] and str(uuid.UUID(row["session_id"], version=4)) == row["session_id"]
+ ), "session id is not valid uuid"
+ assert row["name"] == agent_name
+ assert row["class"] == "AssistantAgent"
+ assert row["init_args"] == json.dumps(init_args)
+
+
+def test_log_oai_wrapper(db_connection):
+ from autogen import OpenAIWrapper
+
+ cur = db_connection.cursor()
+
+ llm_config = {"config_list": [{"model": "gpt-4", "api_key": "some_key", "base_url": "some url"}]}
+ init_args = {"llm_config": llm_config, "base_config": {}}
+ wrapper = OpenAIWrapper(**llm_config)
+
+ autogen.runtime_logging.log_new_wrapper(wrapper, init_args)
+
+ query = """
+ SELECT session_id, init_args FROM oai_wrappers
+ """
+
+ for row in cur.execute(query):
+ assert (
+ row["session_id"] and str(uuid.UUID(row["session_id"], version=4)) == row["session_id"]
+ ), "session id is not valid uuid"
+ saved_init_args = json.loads(row["init_args"])
+ assert "config_list" in saved_init_args
+ assert "api_key" not in saved_init_args["config_list"][0]
+ assert "base_url" not in saved_init_args["config_list"][0]
+ assert "base_config" in saved_init_args
+
+
+def test_log_oai_client(db_connection):
+ cur = db_connection.cursor()
+
+ openai_config = {
+ "api_key": "some_key",
+ "api_version": "2023-12-01-preview",
+ "azure_deployment": "gpt-4",
+ "azure_endpoint": "https://foobar.openai.azure.com/",
+ }
+ client = AzureOpenAI(**openai_config)
+
+ autogen.runtime_logging.log_new_client(client, Mock(), openai_config)
+
+ query = """
+ SELECT session_id, init_args, class FROM oai_clients
+ """
+
+ for row in cur.execute(query):
+ assert (
+ row["session_id"] and str(uuid.UUID(row["session_id"], version=4)) == row["session_id"]
+ ), "session id is not valid uuid"
+ assert row["class"] == "AzureOpenAI"
+ saved_init_args = json.loads(row["init_args"])
+ assert "api_version" in saved_init_args
+ assert "api_key" not in saved_init_args
+
+
+def test_to_dict():
+ from autogen import Agent
+
+ agent1 = autogen.ConversableAgent(
+ "alice",
+ human_input_mode="NEVER",
+ llm_config=False,
+ default_auto_reply="This is alice speaking.",
+ )
+
+ agent2 = autogen.ConversableAgent(
+ "bob",
+ human_input_mode="NEVER",
+ llm_config=False,
+ default_auto_reply="This is bob speaking.",
+ function_map={"test_func": lambda x: x},
+ )
+
+ class Foo:
+ def __init__(self):
+ self.a = 1.234
+ self.b = "some string"
+ self.c = {"some_key": [7, 8, 9]}
+ self.d = None
+ self.test_function = lambda x, y: x + y
+ self.extra_key = "remove this key"
+
+ class Bar(object):
+ def init(self):
+ pass
+
+ def build(self):
+ self.foo_val = [Foo()]
+ self.o = {"key_1": None, "key_2": [{"nested_key_1": ["nested_val_1", "nested_val_2"]}]}
+ self.agents = [agent1, agent2]
+ self.first_agent = agent1
+
+ bar = Bar()
+ bar.build()
+
+ expected_foo_val_field = [
+ {
+ "a": 1.234,
+ "b": "some string",
+ "c": {"some_key": [7, 8, 9]},
+ "d": None,
+ "test_function": "self.test_function = lambda x, y: x + y",
+ }
+ ]
+
+ expected_o_field = {"key_2": [{"nested_key_1": ["nested_val_1", "nested_val_2"]}]}
+
+ result = to_dict(bar, exclude=("key_1", "extra_key"), no_recursive=(Agent))
+ assert result["foo_val"] == expected_foo_val_field
+ assert result["o"] == expected_o_field
+ assert len(result["agents"]) == 2
+ for agent in result["agents"]:
+ assert "autogen.agentchat.conversable_agent.ConversableAgent" in agent
+ assert "autogen.agentchat.conversable_agent.ConversableAgent" in result["first_agent"]
+
+
+@patch("logging.Logger.error")
+def test_logging_exception_will_not_crash_only_print_error(mock_logger_error, db_connection):
+ sample_completion = get_sample_chat_completion(SAMPLE_CHAT_REQUEST)
+ sample_completion["is_cached"] = {"foo": "bar"}
+
+ autogen.runtime_logging.log_chat_completion(**sample_completion)
+
+ args, _ = mock_logger_error.call_args
+ error_message = args[0]
+ assert error_message.startswith("[SqliteLogger] log_chat_completion error:")
diff --git a/website/docs/Migration-Guide.md b/website/docs/Migration-Guide.md
index 178e7a7be..f996a157a 100644
--- a/website/docs/Migration-Guide.md
+++ b/website/docs/Migration-Guide.md
@@ -16,7 +16,18 @@ response = client.create(messages=[{"role": "user", "content": "2+2="}])
print(client.extract_text_or_completion_object(response))
```
-- Inference parameter tuning and inference logging features are currently unavailable in `OpenAIWrapper`. Logging will be added in a future release.
+- Inference parameter tuning and inference logging features are updated:
+```python
+import autogen.runtime_logging
+
+# Start logging
+autogen.runtime_logging.start()
+
+# Stop logging
+autogen.runtime_logging.stop()
+```
+Checkout [Logging documentation](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference#logging) and [Logging example notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_logging.ipynb) to learn more.
+
Inference parameter tuning can be done via [`flaml.tune`](https://microsoft.github.io/FLAML/docs/Use-Cases/Tune-User-Defined-Function).
- `seed` in autogen is renamed into `cache_seed` to accommodate the newly added `seed` param in openai chat completion api. `use_cache` is removed as a kwarg in `OpenAIWrapper.create()` for being automatically decided by `cache_seed`: int | None. The difference between autogen's `cache_seed` and openai's `seed` is that:
- autogen uses local disk cache to guarantee the exactly same output is produced for the same input and when cache is hit, no openai api call will be made.
diff --git a/website/docs/Use-Cases/enhanced_inference.md b/website/docs/Use-Cases/enhanced_inference.md
index ba5153a34..f8cb40fc7 100644
--- a/website/docs/Use-Cases/enhanced_inference.md
+++ b/website/docs/Use-Cases/enhanced_inference.md
@@ -368,10 +368,91 @@ context.append(
)
response = client.create(context=context, messages=messages, **config)
```
+## Logging
-## Logging (for openai<1)
+When debugging or diagnosing an LLM-based system, it is often convenient to log the API calls and analyze them.
-When debugging or diagnosing an LLM-based system, it is often convenient to log the API calls and analyze them. `autogen.Completion` and `autogen.ChatCompletion` offer an easy way to collect the API call histories. For example, to log the chat histories, simply run:
+### For openai >= 1
+
+Logging example: [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_logging.ipynb)
+
+#### Start logging:
+```python
+import autogen.runtime_logging
+
+autogen.runtime_logging.start(logger_type="sqlite", config={"dbname": "YOUR_DB_NAME"})
+```
+`logger_type` and `config` are both optional. Default logger type is SQLite logger, that's the only one available in autogen at the moment. If you want to customize the database name, you can pass in through config, default is `logs.db`.
+
+#### Stop logging:
+```python
+autogen.runtime_logging.stop()
+```
+
+#### LLM Runs
+
+AutoGen logging supports OpenAI's llm message schema. Each LLM run is saved in `chat_completions` table includes:
+- session_id: an unique identifier for the logging session
+- invocation_id: an unique identifier for the logging record
+- client_id: an unique identifier for the Azure OpenAI/OpenAI client
+- request: detailed llm request, see below for an example
+- response: detailed llm response, see below for an example
+- cost: total cost for the request and response
+- start_time
+- end_time
+
+##### Sample Request
+```json
+{
+ "messages":[
+ {
+ "content":"system_message_1",
+ "role":"system"
+ },
+ {
+ "content":"user_message_1",
+ "role":"user"
+ }
+ ],
+ "model":"gpt-4",
+ "temperature": 0.9
+}
+```
+
+##### Sample Response
+```json
+{
+ "id": "id_1",
+ "choices": [
+ {
+ "finish_reason": "stop",
+ "index": 0,
+ "logprobs": null,
+ "message": {
+ "content": "assistant_message_1",
+ "role": "assistant",
+ "function_call": null,
+ "tool_calls": null
+ }
+ }
+ ],
+ "created": "<timestamp>",
+ "model": "gpt-4",
+ "object": "chat.completion",
+ "system_fingerprint": null,
+ "usage": {
+ "completion_tokens": 155,
+ "prompt_tokens": 53,
+ "total_tokens": 208
+ }
+}
+```
+
+Learn more about [request and response format](https://platform.openai.com/docs/api-reference/chat/create)
+
+### For openai < 1
+
+`autogen.Completion` and `autogen.ChatCompletion` offer an easy way to collect the API call histories. For example, to log the chat histories, simply run:
```python
autogen.ChatCompletion.start_logging()
```