Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Normal exits return success, Remove Logger, Update tests (#114)
Shawn Qiu3a691d5cfa23
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex ce32404a..27478f35 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -2,7 +2,6 @@from .client import Clientfrom .event import Event, ActionEvent, LLMEvent, ToolEvent, ErrorEvent-from .logger import AgentOpsLoggerfrom .enums import Models, LLMMessageFormatfrom .decorators import record_functiondiff --git a/agentops/client.py b/agentops/client.pyindex cd9da382..dd95ff78 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -236,10 +236,10 @@ class Client(metaclass=MetaClient):self._worker.create_agent(agent_id, name)def _handle_unclean_exits(self):- def cleanup(end_state_reason: Optional[str] = None):+ def cleanup(end_state: Optional[str] = 'Fail', end_state_reason: Optional[str] = None):# Only run cleanup function if session is createdif self._session is not None:- self.end_session(end_state='Fail',+ self.end_session(end_state=end_state,end_state_reason=end_state_reason)def signal_handler(signum, frame):@@ -270,14 +270,13 @@ class Client(metaclass=MetaClient):formatted_traceback = ''.join(traceback.format_exception(exc_type, exc_value,exc_traceback))- # Perform cleanup- cleanup(- end_state_reason=f"{str(exc_value)}: {formatted_traceback}")+ self.end_session(end_state='Fail',+ end_state_reason=f"{str(exc_value)}: {formatted_traceback}")# Then call the default excepthook to exit the programsys.__excepthook__(exc_type, exc_value, exc_traceback)- atexit.register(lambda: cleanup())+ atexit.register(lambda: cleanup(end_state="Success", end_state_reason="Process exited normally"))signal.signal(signal.SIGINT, signal_handler)signal.signal(signal.SIGTERM, signal_handler)sys.excepthook = handle_exceptiondiff --git a/agentops/logger.py b/agentops/logger.pydeleted file mode 100755index dd472377..00000000--- a/agentops/logger.py+++ /dev/null@@ -1,112 +0,0 @@-import logging-import re-from .client import Client-from .event import Event---class AgentOpsLogger():- """- A utility class for creating loggers and handlers configured to work with the AgentOps service.-- This class provides two static methods for creating a logger or a handler that sends log- records to the AgentOps service. The logger and handler are configured with a specific- AgentOps client and name.-- Example Usage:-- >>> from agentops import Client- >>> client = Client(...)- >>> logger = AgentOpsLogger.get_agentops_logger(client, 'my_logger')- >>> logger.info('This is an info log')-- This will send an 'info' log to the AgentOps service.- """-- @staticmethod- def get_agentops_logger(client: Client, name: str, level=logging.DEBUG):- """- Create and return a logger with an AgentOpsHandler.-- Args:- client (Client): The AgentOps client to which the logs will be sent.- name (str): The name for the logger and handler.- level (int, optional): The minimum severity level to log. Defaults to logging.DEBUG.-- Returns:- logging.Logger: A logger configured with an AgentOpsHandler.- """- logger = logging.getLogger(name)- logger.setLevel(level)- handler = AgentOpsHandler(client, name)- handler.setLevel(level)- logger.addHandler(handler)- return logger-- @staticmethod- def get_agentops_handler(client: Client, name: str):- """- Create and return an AgentOpsHandler.-- Args:- client (Client): The AgentOps client to which the logs will be sent.- name (str): The name for the handler.-- Returns:- AgentOpsHandler: A new AgentOpsHandler with the given client and name.- """- return AgentOpsHandler(client, name)---class AgentOpsHandler(logging.Handler):- """- Custom logging handler for sending logs to the AgentOps service.-- This handler extends the built-in logging.Handler class to send log records to AgentOps.- It also removes ANSI color codes from log messages before sending them.- """-- def __init__(self, client: Client, name: str):- """- Initialize the handler with a specific AgentOps client and name.-- Args:- client (Client): The AgentOps client to which the logs will be sent.- name (str): The name for this handler.- """- super().__init__()- self.name = name- self.client = client-- @staticmethod- def remove_color_codes(s: str) -> str:- """- Remove ANSI color codes from a string.-- Args:- s (str): The string from which color codes will be removed.-- Returns:- The same string, but without any color codes.- """- ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")- return ansi_escape.sub("", s)-- def emit(self, record):- """- Process a log record and send it to the AgentOps client.-- This method is called whenever a log record needs to be processed.-- Args:- record (logging.LogRecord): The log record to process.- """- log_entry = self.format(record)- log_entry = self.remove_color_codes(log_entry)-- if record.levelno == logging.ERROR:- result = "Fail"- else:- result = 'Indeterminate'-- self.client.record(- Event(f'{self.name}:{record.levelname}', returns=log_entry, result=result))diff --git a/tests/api_call_agent.py b/tests/api_call_agent.pydeleted file mode 100644index 339d12ab..00000000--- a/tests/api_call_agent.py+++ /dev/null@@ -1,18 +0,0 @@-import openai-from openai import ChatCompletion--import agentops--ao_client = agentops.Client(api_key='',- endpoint="http://localhost:8000",- tags=['mock tests'])--openai.api_key = ''--message = [{"role": "user", "content": "Hello"},- {"role": "assistant", "content": "Hi there!"}]--ChatCompletion.create(- model='gpt-3.5-turbo', messages=message, temperature=0.5)--ao_client.end_session('Success')diff --git a/tests/fail_agent.py b/tests/fail_agent.pydeleted file mode 100644index ed18543b..00000000--- a/tests/fail_agent.py+++ /dev/null@@ -1,39 +0,0 @@-import agentops-import time-import asyncio-from dotenv import load_dotenv--load_dotenv()---print('init')--ao_client = agentops.Client(tags=['mock tests'])---@ao_client.record_action('fail')-def fail_func(sleep):- time.sleep(sleep)- print('sync sleep')-- print(1/0)---@ao_client.record_action('sleep')-def sleep_func(sleep):- time.sleep(sleep)- print('sync sleep')---async def main():-- print('Action 1')- sleep_func(0.1)- fail_func(0.1)-- ao_client.end_session(end_state='Success')---if __name__ == '__main__':- asyncio.run(main())- print('done')diff --git a/tests/mock_agent.py b/tests/mock_agent.pydeleted file mode 100644index d7fcee75..00000000--- a/tests/mock_agent.py+++ /dev/null@@ -1,70 +0,0 @@-import agentops-import time-import asyncio-import os-from dotenv import load_dotenv--load_dotenv()---print('init')--ao_client = agentops.Client(tags=['mock tests'])---@ao_client.record_action('action')-def sleep_func(sleep):- time.sleep(sleep)- print('sync sleep')- try:- raise ValueError- except:- ...---@ao_client.record_action('async')-async def sleep_func_async(sleep):- await asyncio.sleep(sleep)- print('async sleep')- try:- raise ValueError- except:- ...---@ao_client.record_action('multi')-def multi_event(sleep):- sleep_func(1)- time.sleep(sleep)---async def main():-- print('Action 1')- try:- sleep_func(0.1)- except:- pass-- try:- multi_event(3)- except:- pass-- task1 = asyncio.create_task(sleep_func_async(3))- task2 = asyncio.create_task(sleep_func_async(3))- await task1- await task2-- print('Action 1')- try:- sleep_func(0.1)- except:- pass- print('End Session')- ao_client.end_session(end_state='Success')---if __name__ == '__main__':- asyncio.run(main())- print('done')diff --git a/tests/openai_handlers/_test_handler_openai_v1.py b/tests/openai_handlers/_test_handler_openai_v1.pyindex eed73522..97ea1795 100644--- a/tests/openai_handlers/_test_handler_openai_v1.py+++ b/tests/openai_handlers/_test_handler_openai_v1.py@@ -1,6 +1,5 @@import timefrom openai.resources.chat import completions-import openaifrom openai import OpenAI, AsyncOpenAIimport agentopsimport asyncio@@ -15,8 +14,8 @@ async_client = AsyncOpenAI()print('Running OpenAI v1.0.0+')-# Assuming that instantiating agentops.Client will trigger the LlmTracker to override methods-ao_client = agentops.Client(tags=['mock agent', openai.__version__])+# Assuming that initializing will trigger the LlmTracker to override methods+agentops.init(tags=['mock agent', openai.__version__])# Now the client.chat.completions.create should be the overridden method@@ -94,4 +93,4 @@ asyncio.run(test_async_chat_completion())print('Running async streaming test')asyncio.run(test_async_chat_completion_stream())-ao_client.end_session('Success')+agentops.end_session('Success')diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_async.py b/tests/openai_handlers/_test_llm_tracker_ge_1_async.pysimilarity index 100%rename from tests/openai_handlers/test_llm_tracker_ge_1_async.pyrename to tests/openai_handlers/_test_llm_tracker_ge_1_async.pydiff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/_test_llm_tracker_ge_1_sync.pysimilarity index 100%rename from tests/openai_handlers/test_llm_tracker_ge_1_sync.pyrename to tests/openai_handlers/_test_llm_tracker_ge_1_sync.pydiff --git a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/_test_llm_tracker_lt_1_sync.pysimilarity index 100%rename from tests/openai_handlers/test_llm_tracker_lt_1_sync.pyrename to tests/openai_handlers/_test_llm_tracker_lt_1_sync.pydiff --git a/tests/test_canary.py b/tests/test_canary.pyindex ddceba50..48a37b13 100644--- a/tests/test_canary.py+++ b/tests/test_canary.py@@ -1,8 +1,8 @@import pytestimport requests_mockimport time--from agentops import Client, Event+import agentops+from agentops import ActionEvent@pytest.fixture@@ -18,17 +18,17 @@ class TestCanary:def setup_method(self):self.url = 'https://api.agentops.ai'self.api_key = "random_api_key"- self.client = Client(api_key=self.api_key, max_wait_time=5)+ agentops.init(api_key=self.api_key, max_wait_time=5)def teardown_method(self):- self.client.end_session(end_state='Success')+ agentops.end_session(end_state='Success')def test_agent_ops_record(self, mock_req):# Arrangeevent_type = 'test_event_type'# Act- self.client.record(Event(event_type))+ agentops.record(ActionEvent(event_type))time.sleep(0.1)# Assertdiff --git a/tests/test_logger.py b/tests/test_logger.pydeleted file mode 100755index 09b1835e..00000000--- a/tests/test_logger.py+++ /dev/null@@ -1,89 +0,0 @@-import pytest-import requests_mock-import time-from agentops import Client, AgentOpsLogger---def mock_req():- with requests_mock.Mocker() as m:- url = 'https://api.agentops.ai'- m.post(url + '/events', text='ok')- m.post(url + '/sessions', text='ok')- yield m---class TestLogger:- def setup_method(self):- self.url = 'https://api.agentops.ai'- self.api_key = "random_api_key"- self.client = Client(api_key=self.api_key, max_wait_time=5)-- def teardown_method(self):- self.client.end_session(end_state='Success')-- def test_info(self, mock_req):- # Arrange- self.event_type = "info"- self.logger = AgentOpsLogger.get_agentops_logger(- self.client, self.event_type)- test_message = "Test info message"-- # Act- try:- self.logger.info(test_message)- except Exception as e:- pytest.fail(f"test_info failed with {e}")-- time.sleep(0.1)-- # Assert- assert len(mock_req.request_history) == 1- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key- request_json = mock_req.last_request.json()- assert request_json['events'][0]['event_type'] == f"{self.event_type}:INFO"- assert request_json['events'][0]['returns'] == test_message-- def test_error(self, mock_req):- # Arrange- test_message = "Test error message"- self.event_type = "error"- self.logger = AgentOpsLogger.get_agentops_logger(- self.client, self.event_type)-- # Act- try:- self.logger.error(test_message)- except Exception as e:- pytest.fail(f"test_error failed with {e}")-- time.sleep(0.1)-- # Assert- assert len(mock_req.request_history) == 1- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key- request_json = mock_req.last_request.json()- assert request_json['events'][0]['event_type'] == f"{self.event_type}:ERROR"- assert request_json['events'][0]['returns'] == test_message-- def test_warn(self, mock_req):- # Arrange- test_message = "Test warn message"- self.event_type = "warning"- self.logger = AgentOpsLogger.get_agentops_logger(- self.client, self.event_type)-- # Act- try:- self.logger.warning(test_message)- except Exception as e:- pytest.fail(f"test_warn failed with {e}")-- time.sleep(0.1)-- # Assert- assert len(mock_req.request_history) == 1- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key- request_json = mock_req.last_request.json()- assert request_json['events'][0]['event_type'] == f"{self.event_type}:WARNING"- assert request_json['events'][0]['returns'] == test_messagediff --git a/tests/test_record_function.py b/tests/test_record_function.pynew file mode 100644index 00000000..de03a81f--- /dev/null+++ b/tests/test_record_function.py@@ -0,0 +1,90 @@+import pytest+import requests_mock+import time+import agentops+from agentops import record_function+from datetime import datetime++def mock_req():+ with requests_mock.Mocker() as m:+ url = 'https://api.agentops.ai'+ m.post(url + '/events', text='ok')+ m.post(url + '/sessions', text='ok')+ yield m++class TestRecordAction:+ def setup_method(self):+ self.url = 'https://api.agentops.ai'+ self.api_key = "random_api_key"+ self.event_type = 'test_event_type'+ agentops.init(self.api_key, max_wait_time=5, auto_start_session=False)+ agentops.start_session()++ def teardown_method(self):+ agentops.end_session(end_state='Success')++ def test_record_function_decorator(self, mock_req):+ @record_function(event_name=self.event_type)+ def add_two(x, y):+ return x + y++ # Act+ add_two(3, 4)+ time.sleep(0.1)++ # Assert+ assert len(mock_req.request_history) == 1+ assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key+ request_json = mock_req.last_request.json()+ assert request_json['events'][0]['action_type'] == self.event_type+ assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}+ assert request_json['events'][0]['returns'] == 7++ def test_record_function_decorator_multiple(self, mock_req):+ # Arrange+ @record_function(event_name=self.event_type)+ def add_three(x, y, z=3):+ return x + y + z++ # Act+ add_three(1, 2)+ time.sleep(0.1)+ add_three(1, 2)+ time.sleep(0.1)++ # Assert+ assert len(mock_req.request_history) == 2+ assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key+ request_json = mock_req.last_request.json()+ assert request_json['events'][0]['action_type'] == self.event_type+ assert request_json['events'][0]['params'] == {'x': 1, 'y': 2, 'z': 3}+ assert request_json['events'][0]['returns'] == 6++ @pytest.mark.asyncio+ async def test_async_function_call(self, mock_req):++ @record_function(self.event_type)+ async def async_add(x, y):+ time.sleep(0.1)+ return x + y++ # Act+ result = await async_add(3, 4)+ time.sleep(0.1)++ # Assert+ assert result == 7+ # Assert+ assert len(mock_req.request_history) == 1+ assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key+ request_json = mock_req.last_request.json()+ assert request_json['events'][0]['action_type'] == self.event_type+ assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}+ assert request_json['events'][0]['returns'] == 7+ init = datetime.fromisoformat(+ request_json['events'][0]['init_timestamp'].replace('Z', '+00:00'))+ end = datetime.fromisoformat(+ request_json['events'][0]['end_timestamp'].replace('Z', '+00:00'))++ assert (end - init).total_seconds() >= 0.1\ No newline at end of filediff --git a/tests/test_session.py b/tests/test_session.pyindex 128b9c49..b61e715f 100644--- a/tests/test_session.py+++ b/tests/test_session.py@@ -1,10 +1,8 @@import pytestimport requests_mockimport time-from datetime import datetime---from agentops import Client, Event, record_function+import agentops+from agentops import ActionEvent@pytest.fixture@@ -20,19 +18,20 @@ class TestSessions:def setup_method(self):self.api_key = "random_api_key"self.event_type = 'test_event_type'+ agentops.init(api_key=self.api_key, max_wait_time=5, auto_start_session=False)def test_session(self, mock_req):# Arrange- client = Client(api_key=self.api_key, max_wait_time=5)+ agentops.start_session()# Act- client.record(Event(self.event_type))+ agentops.record(ActionEvent(self.event_type))# Assert the session has been initiated and the id has been created on backend.assert len(mock_req.request_history) == 1# Act- client.record(Event(self.event_type))+ agentops.record(ActionEvent(self.event_type))time.sleep(0.1)# Assert an event has been added@@ -43,24 +42,23 @@ class TestSessions:# Actend_state = 'Success'- client.end_session(end_state)+ agentops.end_session(end_state)time.sleep(0.1)# Since a session has ended, no more events should be recorded, but end_session should be calledassert len(mock_req.request_history) == 3assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()- assert request_json['session']['rating'] == Noneassert request_json['session']['end_state'] == end_stateassert request_json['session']['tags'] == Nonedef test_tags(self, mock_req):# Arrangetags = ['GPT-4']- client = Client(api_key=self.api_key, tags=tags, max_wait_time=5)+ agentops.start_session(tags=tags)# Act- client.record(Event(self.event_type))+ agentops.record(ActionEvent(self.event_type))time.sleep(0.1)# Assert 2 requests - 1 for session init, 1 for event@@ -71,131 +69,12 @@ class TestSessions:# Actend_state = 'Success'- client.end_session(end_state)+ agentops.end_session(end_state)time.sleep(0.1)# Assert 3 requets, 1 for session init, 1 for event, 1 for end sessionassert len(mock_req.request_history) == 3assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()- assert request_json['session']['rating'] == Noneassert request_json['session']['end_state'] == end_state- assert request_json['session']['tags'] == tags---class TestRecordAction:- def setup_method(self):- self.url = 'https://api.agentops.ai'- self.api_key = "random_api_key"- self.event_type = 'test_event_type'- self.client = Client(self.api_key, max_wait_time=5)-- def teardown_method(self):- self.client.end_session(end_state='Success')-- def test_record_function_decorator(self, mock_req):- @record_function(event_name=self.event_type)- def add_two(x, y):- return x + y-- # Act- add_two(3, 4)- time.sleep(0.1)-- # Assert- assert len(mock_req.request_history) == 1- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key- request_json = mock_req.last_request.json()- assert request_json['events'][0]['event_type'] == self.event_type- assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}- assert request_json['events'][0]['returns'] == 7- assert request_json['events'][0]['result'] == 'Success'- assert request_json['events'][0]['tags'] == ['foo', 'bar']-- def test_record_function_decorator_multiple(self, mock_req):- # Arrange- @record_function(event_name=self.event_type)- def add_three(x, y, z=3):- return x + y + z-- # Act- add_three(1, 2)- time.sleep(0.1)-- # Assert- assert len(mock_req.request_history) == 1- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key- request_json = mock_req.last_request.json()- assert request_json['events'][0]['event_type'] == self.event_type- assert request_json['events'][0]['params'] == {'x': 1, 'y': 2, 'z': 3}- assert request_json['events'][0]['returns'] == 6- assert request_json['events'][0]['result'] == 'Success'- assert request_json['events'][0]['tags'] == ['foo', 'bar']-- @pytest.mark.asyncio- async def test_async_function_call(self, mock_req):-- @record_function(self.event_type)- async def async_add(x, y):- time.sleep(0.1)- return x + y-- # Act- result = await async_add(3, 4)- time.sleep(0.1)-- # Assert- assert result == 7- # Assert- assert len(mock_req.request_history) == 1- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key- request_json = mock_req.last_request.json()- assert request_json['events'][0]['event_type'] == self.event_type- assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}- assert request_json['events'][0]['returns'] == 7- assert request_json['events'][0]['result'] == 'Success'- init = datetime.fromisoformat(- request_json['events'][0]['init_timestamp'].replace('Z', '+00:00'))- end = datetime.fromisoformat(- request_json['events'][0]['end_timestamp'].replace('Z', '+00:00'))-- assert (end - init).total_seconds() >= 0.1-- def test_function_call(self, mock_req):- # Arrange- prompt = 'prompt'-- @record_function(event_name=self.event_type)- def foo(prompt=prompt):- return 'output'-- # Act- foo()- time.sleep(0.1)-- # Assert- assert len(mock_req.request_history) == 1- request_json = mock_req.last_request.json()- assert request_json['events'][0]['action_type'] == 'action'- assert request_json['events'][0]['prompt'] == None- assert request_json['events'][0]['returns'] == 'output'- assert request_json['events'][0]['result'] == 'Success'-- def test_llm_call_no_action_type(self, mock_req):- # Arrange- prompt = 'prompt'-- @record_function(event_name=self.event_type)- def llm_call(prompt=prompt):- return 'output'-- llm_call()- time.sleep(0.1)-- # Assert- assert len(mock_req.request_history) == 1- request_json = mock_req.last_request.json()- assert request_json['events'][0]['action_type'] == 'action'- assert request_json['events'][0]['prompt'] == None- assert request_json['events'][0]['returns'] == 'output'- assert request_json['events'][0]['result'] == 'Success'+ assert request_json['session']['tags'] == tags\ No newline at end of filediff --git a/tox.ini b/tox.iniindex 8f93df07..86213961 100644--- a/tox.ini+++ b/tox.ini@@ -17,6 +17,7 @@ deps =mypy: mypytypes-requestspsutil+ openaicommands =coverage run --source agentops -m pytestcoverage report -m