Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
feature: better logging (#215)
Shawn Qiufb2852bdec58
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 02324a41..46ae55a9 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -1,4 +1,6 @@# agentops/__init__.py+import os+import loggingfrom typing import Optional, List, Unionfrom .client import Clientfrom .config import Configuration@@ -6,7 +8,7 @@ from .event import Event, ActionEvent, LLMEvent, ToolEvent, ErrorEventfrom .enums import Modelsfrom .decorators import record_functionfrom .agent import track_agent-from .log_config import set_logging_level_info, set_logging_level_critial+from .log_config import loggertry:from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandlerexcept ModuleNotFoundError:@@ -46,7 +48,13 @@ def init(api_key: Optional[str] = None,inherited_session_id (optional, str): Init Agentops with an existing SessionAttributes:"""- set_logging_level_info()+ if os.getenv('AGENTOPS_LOGGING_LEVEL') == 'DEBUG':+ logger.setLevel(logging.DEBUG)+ elif os.getenv('AGENTOPS_LOGGING_LEVEL') == 'CRITICAL':+ logger.setLevel(logging.CRITICAL)+ else:+ logger.setLevel(logging.INFO)+c = Client(api_key=api_key,parent_key=parent_key,endpoint=endpoint,@@ -131,6 +139,7 @@ def set_parent_key(parent_key):"""Client().set_parent_key(parent_key)+def stop_instrumenting():Client().stop_instrumenting()diff --git a/agentops/agent.py b/agentops/agent.pyindex 0e070b62..9519e73b 100644--- a/agentops/agent.py+++ b/agentops/agent.py@@ -20,7 +20,7 @@ def track_agent(name: Union[str, None] = None):self.agent_ops_agent_id = str(uuid4())Client().create_agent(name=self.agent_ops_agent_name, agent_id=self.agent_ops_agent_id)except AttributeError as e:- logger.warning("AgentOps failed to track an agent. This often happens if agentops.init() was not "+ logger.warning("Failed to track an agent. This often happens if agentops.init() was not ""called before initializing an agent with the @track_agent decorator.")raise e@@ -31,7 +31,8 @@ def track_agent(name: Union[str, None] = None):Client().create_agent(name=obj.agent_ops_agent_name, agent_id=obj.agent_ops_agent_id)else:- raise Exception("Invalid input, 'obj' must be a class or a function")+ raise Exception(+ "Invalid input, 'obj' must be a class or a function")return objdiff --git a/agentops/client.py b/agentops/client.pyindex 2d8abd96..22103c12 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -5,7 +5,16 @@Client: Provides methods to interact with the AgentOps service."""import os-import uuid+import inspect+import atexit+import signal+import sys+import threading+import traceback+import logging+from decimal import Decimal+from uuid import UUID, uuid4+from typing import Optional, List, Unionfrom .event import ActionEvent, ErrorEvent, Eventfrom .enums import EndState@@ -13,18 +22,7 @@ from .helpers import get_ISO_time, singleton, check_call_stack_for_agent_id, getfrom .session import Sessionfrom .worker import Workerfrom .host_env import get_host_env-from uuid import uuid4-from typing import Optional, List, Union-import traceback-from .log_config import logger, set_logging_level_info-from decimal import Decimal-import inspect-import atexit-import signal-import sys-import threading--+from .log_config import loggerfrom .meta_client import MetaClientfrom .config import Configuration, ConfigurationErrorfrom .llm_tracker import LlmTracker@@ -73,7 +71,7 @@ class Client(metaclass=MetaClient):):if override is not None:- logger.warning("🖇 AgentOps: The 'override' parameter is deprecated. Use 'instrument_llm_calls' instead.",+ logger.warning("The 'override' parameter is deprecated. Use 'instrument_llm_calls' instead.",DeprecationWarning, stacklevel=2)instrument_llm_calls = instrument_llm_calls or override@@ -120,9 +118,8 @@ class Client(metaclass=MetaClient):except ImportError:passexcept Exception as e:- logger.warning("🖇️ AgentOps: Failed to set up autogen logger with AgentOps. Error: " + e)-- return partner_frameworks[framework]+ logger.warning(+ "Failed to set up autogen logger with AgentOps. Error: " + e)return instrument_llm_calls, auto_start_session@@ -173,8 +170,8 @@ class Client(metaclass=MetaClient):Args:event (Event): The event to record."""- if self._session is None or self._session.has_ended:- logger.warning("🖇 AgentOps: Cannot record event - no current session")+ if self._session is None or self._session.has_ended or self._worker is None:+ logger.warning("Cannot record event - no current session")returnif isinstance(event, Event):@@ -274,13 +271,21 @@ class Client(metaclass=MetaClient):config: (Configuration, optional): Client configuration objectinherited_session_id (optional, str): assign session id to match existing Session"""- set_logging_level_info()+ if os.getenv('AGENTOPS_LOGGING_LEVEL') == 'DEBUG':+ logger.setLevel(logging.DEBUG)+ elif os.getenv('AGENTOPS_LOGGING_LEVEL') == 'CRITICAL':+ logger.setLevel(logging.CRITICAL)+ else:+ logger.setLevel(logging.INFO)if self._session is not None:- return logger.warning("🖇 AgentOps: Cannot start session - session already started")+ return logger.warning("Cannot start session - session already started")if not config and not self.config:- return logger.warning("🖇 AgentOps: Cannot start session - missing configuration")+ return logger.warning("Cannot start session - missing configuration")++ session_id = UUID(+ inherited_session_id) if inherited_session_id is not None else uuid4()self._session = Session(inherited_session_id or uuid4(),tags or self._tags_for_future_session, host_env=get_host_env(self._env_data_opt_out))@@ -288,9 +293,9 @@ class Client(metaclass=MetaClient):start_session_result = self._worker.start_session(self._session)if not start_session_result:self._session = None- return logger.warning("🖇 AgentOps: Cannot start session - No server response")+ return logger.warning("Cannot start session - No server response")- logger.info('View info on this session at https://app.agentops.ai/drilldown?session_id=%s',+ logger.info('\x1b[34mView info on this session at https://app.agentops.ai/drilldown?session_id=%s\x1b[0m',self._session.session_id)return self._session.session_id@@ -308,23 +313,23 @@ class Client(metaclass=MetaClient):video (str, optional): The video screen recording of the session"""if self._session is None or self._session.has_ended:- return logger.warning("🖇 AgentOps: Cannot end session - no current session")+ return logger.warning("Cannot end session - no current session")if not any(end_state == state.value for state in EndState):- return logger.warning("🖇 AgentOps: Invalid end_state. Please use one of the EndState enums")+ return logger.warning("Invalid end_state. Please use one of the EndState enums")if self._worker is None or self._worker._session is None:- return logger.warning("🖇 AgentOps: Cannot end session - no current worker or session")+ return logger.warning("Cannot end session - no current worker or session")self._session.video = videoself._session.end_session(end_state, end_state_reason)token_cost = self._worker.end_session(self._session)if token_cost == 'unknown':- print('🖇 AgentOps: Could not determine cost of run.')+ logger.info('Could not determine cost of run.')else:token_cost_d = Decimal(token_cost)- print('\n🖇 AgentOps: This run cost ${}'.format('{:.2f}'.format(+ logger.info('This run cost ${}'.format('{:.2f}'.format(token_cost_d) if token_cost_d == 0 else '{:.6f}'.format(token_cost_d)))self._session = Noneself._worker = None@@ -352,7 +357,7 @@ class Client(metaclass=MetaClient):frame: The current stack frame."""signal_name = 'SIGINT' if signum == signal.SIGINT else 'SIGTERM'- logger.info('🖇 AgentOps: %s detected. Ending session...', signal_name)+ logger.info('%s detected. Ending session...', signal_name)self.end_session(end_state='Fail',end_state_reason=f'Signal {signal_name} detected')sys.exit(0)diff --git a/agentops/config.py b/agentops/config.pyindex 6ea2f7db..e2681fb8 100644--- a/agentops/config.py+++ b/agentops/config.py@@ -35,13 +35,15 @@ class Configuration:if not api_key:api_key = environ.get('AGENTOPS_API_KEY', None)if not api_key:- raise ConfigurationError("🖇 AgentOps: No API key provided - no data will be recorded.")+ raise ConfigurationError(+ "No API key provided - no data will be recorded.")if not parent_key:parent_key = environ.get('AGENTOPS_PARENT_KEY', None)if not endpoint:- endpoint = environ.get('AGENTOPS_API_ENDPOINT', 'https://api.agentops.ai')+ endpoint = environ.get(+ 'AGENTOPS_API_ENDPOINT', 'https://api.agentops.ai')self._api_key: str = api_keyself._endpoint = endpointdiff --git a/agentops/http_client.py b/agentops/http_client.pyindex fee12f98..a007fe1a 100644--- a/agentops/http_client.py+++ b/agentops/http_client.py@@ -81,7 +81,7 @@ class HttpClient:result.code = 408result.status = HttpStatus.TIMEOUTlogger.warning(- '🖇 AgentOps: Could not post data - connection timed out')+ 'Could not post data - connection timed out')except requests.exceptions.HTTPError as e:try:result.parse(e.response)@@ -94,12 +94,12 @@ class HttpClient:result.body = {'error': str(e)}if result.code == 401:- logger.warning('🖇 AgentOps: Could not post data - API server rejected your API key: %s',+ logger.warning('Could not post data - API server rejected your API key: %s',api_key)if result.code == 400:- logger.warning('🖇 AgentOps: Could not post data - %s', result.body)+ logger.warning('Could not post data - %s', result.body)if result.code == 500:logger.warning(- '🖇 AgentOps: Could not post data - internal server error')+ 'Could not post data - internal server error')return resultdiff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex fd60f67d..78f25031 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -80,7 +80,7 @@ class LlmTracker:kwargs_str = pprint.pformat(kwargs)chunk = pprint.pformat(chunk)logger.warning(- f"🖇 AgentOps: Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n"+ f"Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n"f"chunk:\n {chunk}\n"f"kwargs:\n {kwargs_str}\n")@@ -119,7 +119,7 @@ class LlmTracker:kwargs_str = pprint.pformat(kwargs)response = pprint.pformat(response)logger.warning(- f"🖇 AgentOps: Unable to parse response for LLM call. Skipping upload to AgentOps\n"+ f"Unable to parse response for LLM call. Skipping upload to AgentOps\n"f"response:\n {response}\n"f"kwargs:\n {kwargs_str}\n")@@ -175,7 +175,7 @@ class LlmTracker:kwargs_str = pprint.pformat(kwargs)chunk = pprint.pformat(chunk)logger.warning(- f"🖇 AgentOps: Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n"+ f"Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n"f"chunk:\n {chunk}\n"f"kwargs:\n {kwargs_str}\n")@@ -220,7 +220,7 @@ class LlmTracker:kwargs_str = pprint.pformat(kwargs)response = pprint.pformat(response)logger.warning(- f"🖇 AgentOps: Unable to parse response for LLM call. Skipping upload to AgentOps\n"+ f"Unable to parse response for LLM call. Skipping upload to AgentOps\n"f"response:\n {response}\n"f"kwargs:\n {kwargs_str}\n")@@ -323,7 +323,7 @@ class LlmTracker:kwargs_str = pprint.pformat(kwargs)chunk = pprint.pformat(chunk)logger.warning(- f"🖇 AgentOps: Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n"+ f"Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n"f"chunk:\n {chunk}\n"f"kwargs:\n {kwargs_str}\n")@@ -376,7 +376,7 @@ class LlmTracker:kwargs_str = pprint.pformat(kwargs)response = pprint.pformat(response)logger.warning(- f"🖇 AgentOps: Unable to parse response for LLM call. Skipping upload to AgentOps\n"+ f"Unable to parse response for LLM call. Skipping upload to AgentOps\n"f"response:\n {response}\n"f"kwargs:\n {kwargs_str}\n")@@ -516,13 +516,13 @@ class LlmTracker:if api == 'litellm':module_version = version(api)if module_version is None:- logger.warning(f'🖇 AgentOps: Cannot determine LiteLLM version. Only LiteLLM>=1.3.1 supported.')+ logger.warning(f'Cannot determine LiteLLM version. Only LiteLLM>=1.3.1 supported.')if Version(module_version) >= parse('1.3.1'):self.override_litellm_completion()self.override_litellm_async_completion()else:- logger.warning(f'🖇 AgentOps: Only LiteLLM>=1.3.1 supported. v{module_version} found.')+ logger.warning(f'Only LiteLLM>=1.3.1 supported. v{module_version} found.')return # If using an abstraction like litellm, do not patch the underlying LLM APIsif api == 'openai':@@ -541,13 +541,13 @@ class LlmTracker:# Patch cohere v5.4.0+ methodsmodule_version = version(api)if module_version is None:- logger.warning(f'🖇 AgentOps: Cannot determine Cohere version. Only Cohere>=5.4.0 supported.')+ logger.warning(f'Cannot determine Cohere version. Only Cohere>=5.4.0 supported.')if Version(module_version) >= parse('5.4.0'):self.override_cohere_chat()self.override_cohere_chat_stream()else:- logger.warning(f'🖇 AgentOps: Only Cohere>=5.4.0 supported. v{module_version} found.')+ logger.warning(f'Only Cohere>=5.4.0 supported. v{module_version} found.')def stop_instrumenting(self):self.undo_override_openai_v1_async_completion()diff --git a/agentops/log_config.py b/agentops/log_config.pyindex 86cdecaf..8bf5b8eb 100644--- a/agentops/log_config.py+++ b/agentops/log_config.py@@ -1,10 +1,30 @@import logging++class AgentOpsFormatter(logging.Formatter):+ blue = "\x1b[34m"+ bold_red = "\x1b[31;1m"+ reset = "\x1b[0m"+ format = "🖇 AgentOps: %(message)s"++ FORMATS = {+ logging.DEBUG: "(DEBUG) " + format,+ logging.INFO: format,+ logging.WARNING: format,+ logging.ERROR: format,+ logging.CRITICAL: bold_red + format + reset,+ }++ def format(self, record):+ log_fmt = self.FORMATS.get(record.levelno)+ formatter = logging.Formatter(log_fmt)+ return formatter.format(record)++logger = logging.getLogger("agentops")logger.setLevel(logging.CRITICAL)-def set_logging_level_critial():- logger.setLevel(logging.CRITICAL)--def set_logging_level_info():- logger.setLevel(logging.INFO)\ No newline at end of file+handler = logging.StreamHandler()+handler.setLevel(logging.DEBUG)+handler.setFormatter(AgentOpsFormatter())+logger.addHandler(handler)diff --git a/agentops/meta_client.py b/agentops/meta_client.pyindex 68b84060..fe7b5f75 100644--- a/agentops/meta_client.py+++ b/agentops/meta_client.py@@ -48,10 +48,11 @@ def handle_exceptions(method):try:return method(self, *args, **kwargs)except Exception as e:- logger.warning(f"🖇 AgentOps: Error: {e}")+ logger.warning(f"Error: {e}")config = getattr(self, 'config', None)if config is not None:- type(self).send_exception_to_server(e, self.config._api_key, self._session)+ type(self).send_exception_to_server(+ e, self.config._api_key, self._session)raise ereturn wrapper