Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Normal exits return success, Remove Logger, Update tests (#114)

Shawn Qiu3a691d5cfa23
diff --git a/agentops/__init__.py b/agentops/__init__.py
index ce32404a..27478f35 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -2,7 +2,6 @@
from .client import Client
from .event import Event, ActionEvent, LLMEvent, ToolEvent, ErrorEvent
-from .logger import AgentOpsLogger
from .enums import Models, LLMMessageFormat
from .decorators import record_function
diff --git a/agentops/client.py b/agentops/client.py
index cd9da382..dd95ff78 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -236,10 +236,10 @@ class Client(metaclass=MetaClient):
self._worker.create_agent(agent_id, name)
def _handle_unclean_exits(self):
- def cleanup(end_state_reason: Optional[str] = None):
+ def cleanup(end_state: Optional[str] = 'Fail', end_state_reason: Optional[str] = None):
# Only run cleanup function if session is created
if self._session is not None:
- self.end_session(end_state='Fail',
+ self.end_session(end_state=end_state,
end_state_reason=end_state_reason)
def signal_handler(signum, frame):
@@ -270,14 +270,13 @@ class Client(metaclass=MetaClient):
formatted_traceback = ''.join(traceback.format_exception(exc_type, exc_value,
exc_traceback))
- # Perform cleanup
- cleanup(
- end_state_reason=f"{str(exc_value)}: {formatted_traceback}")
+ self.end_session(end_state='Fail',
+ end_state_reason=f"{str(exc_value)}: {formatted_traceback}")
# Then call the default excepthook to exit the program
sys.__excepthook__(exc_type, exc_value, exc_traceback)
- atexit.register(lambda: cleanup())
+ atexit.register(lambda: cleanup(end_state="Success", end_state_reason="Process exited normally"))
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
sys.excepthook = handle_exception
diff --git a/agentops/logger.py b/agentops/logger.py
deleted file mode 100755
index dd472377..00000000
--- a/agentops/logger.py
+++ /dev/null
@@ -1,112 +0,0 @@
-import logging
-import re
-from .client import Client
-from .event import Event
-
-
-class AgentOpsLogger():
- """
- A utility class for creating loggers and handlers configured to work with the AgentOps service.
-
- This class provides two static methods for creating a logger or a handler that sends log
- records to the AgentOps service. The logger and handler are configured with a specific
- AgentOps client and name.
-
- Example Usage:
-
- >>> from agentops import Client
- >>> client = Client(...)
- >>> logger = AgentOpsLogger.get_agentops_logger(client, 'my_logger')
- >>> logger.info('This is an info log')
-
- This will send an 'info' log to the AgentOps service.
- """
-
- @staticmethod
- def get_agentops_logger(client: Client, name: str, level=logging.DEBUG):
- """
- Create and return a logger with an AgentOpsHandler.
-
- Args:
- client (Client): The AgentOps client to which the logs will be sent.
- name (str): The name for the logger and handler.
- level (int, optional): The minimum severity level to log. Defaults to logging.DEBUG.
-
- Returns:
- logging.Logger: A logger configured with an AgentOpsHandler.
- """
- logger = logging.getLogger(name)
- logger.setLevel(level)
- handler = AgentOpsHandler(client, name)
- handler.setLevel(level)
- logger.addHandler(handler)
- return logger
-
- @staticmethod
- def get_agentops_handler(client: Client, name: str):
- """
- Create and return an AgentOpsHandler.
-
- Args:
- client (Client): The AgentOps client to which the logs will be sent.
- name (str): The name for the handler.
-
- Returns:
- AgentOpsHandler: A new AgentOpsHandler with the given client and name.
- """
- return AgentOpsHandler(client, name)
-
-
-class AgentOpsHandler(logging.Handler):
- """
- Custom logging handler for sending logs to the AgentOps service.
-
- This handler extends the built-in logging.Handler class to send log records to AgentOps.
- It also removes ANSI color codes from log messages before sending them.
- """
-
- def __init__(self, client: Client, name: str):
- """
- Initialize the handler with a specific AgentOps client and name.
-
- Args:
- client (Client): The AgentOps client to which the logs will be sent.
- name (str): The name for this handler.
- """
- super().__init__()
- self.name = name
- self.client = client
-
- @staticmethod
- def remove_color_codes(s: str) -> str:
- """
- Remove ANSI color codes from a string.
-
- Args:
- s (str): The string from which color codes will be removed.
-
- Returns:
- The same string, but without any color codes.
- """
- ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
- return ansi_escape.sub("", s)
-
- def emit(self, record):
- """
- Process a log record and send it to the AgentOps client.
-
- This method is called whenever a log record needs to be processed.
-
- Args:
- record (logging.LogRecord): The log record to process.
- """
- log_entry = self.format(record)
- log_entry = self.remove_color_codes(log_entry)
-
- if record.levelno == logging.ERROR:
- result = "Fail"
- else:
- result = 'Indeterminate'
-
- self.client.record(
- Event(f'{self.name}:{record.levelname}', returns=log_entry, result=result))
diff --git a/tests/api_call_agent.py b/tests/api_call_agent.py
deleted file mode 100644
index 339d12ab..00000000
--- a/tests/api_call_agent.py
+++ /dev/null
@@ -1,18 +0,0 @@
-import openai
-from openai import ChatCompletion
-
-import agentops
-
-ao_client = agentops.Client(api_key='',
- endpoint="http://localhost:8000",
- tags=['mock tests'])
-
-openai.api_key = ''
-
-message = [{"role": "user", "content": "Hello"},
- {"role": "assistant", "content": "Hi there!"}]
-
-ChatCompletion.create(
- model='gpt-3.5-turbo', messages=message, temperature=0.5)
-
-ao_client.end_session('Success')
diff --git a/tests/fail_agent.py b/tests/fail_agent.py
deleted file mode 100644
index ed18543b..00000000
--- a/tests/fail_agent.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import agentops
-import time
-import asyncio
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-print('init')
-
-ao_client = agentops.Client(tags=['mock tests'])
-
-
-@ao_client.record_action('fail')
-def fail_func(sleep):
- time.sleep(sleep)
- print('sync sleep')
-
- print(1/0)
-
-
-@ao_client.record_action('sleep')
-def sleep_func(sleep):
- time.sleep(sleep)
- print('sync sleep')
-
-
-async def main():
-
- print('Action 1')
- sleep_func(0.1)
- fail_func(0.1)
-
- ao_client.end_session(end_state='Success')
-
-
-if __name__ == '__main__':
- asyncio.run(main())
- print('done')
diff --git a/tests/mock_agent.py b/tests/mock_agent.py
deleted file mode 100644
index d7fcee75..00000000
--- a/tests/mock_agent.py
+++ /dev/null
@@ -1,70 +0,0 @@
-import agentops
-import time
-import asyncio
-import os
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-print('init')
-
-ao_client = agentops.Client(tags=['mock tests'])
-
-
-@ao_client.record_action('action')
-def sleep_func(sleep):
- time.sleep(sleep)
- print('sync sleep')
- try:
- raise ValueError
- except:
- ...
-
-
-@ao_client.record_action('async')
-async def sleep_func_async(sleep):
- await asyncio.sleep(sleep)
- print('async sleep')
- try:
- raise ValueError
- except:
- ...
-
-
-@ao_client.record_action('multi')
-def multi_event(sleep):
- sleep_func(1)
- time.sleep(sleep)
-
-
-async def main():
-
- print('Action 1')
- try:
- sleep_func(0.1)
- except:
- pass
-
- try:
- multi_event(3)
- except:
- pass
-
- task1 = asyncio.create_task(sleep_func_async(3))
- task2 = asyncio.create_task(sleep_func_async(3))
- await task1
- await task2
-
- print('Action 1')
- try:
- sleep_func(0.1)
- except:
- pass
- print('End Session')
- ao_client.end_session(end_state='Success')
-
-
-if __name__ == '__main__':
- asyncio.run(main())
- print('done')
diff --git a/tests/openai_handlers/_test_handler_openai_v1.py b/tests/openai_handlers/_test_handler_openai_v1.py
index eed73522..97ea1795 100644
--- a/tests/openai_handlers/_test_handler_openai_v1.py
+++ b/tests/openai_handlers/_test_handler_openai_v1.py
@@ -1,6 +1,5 @@
import time
from openai.resources.chat import completions
-import openai
from openai import OpenAI, AsyncOpenAI
import agentops
import asyncio
@@ -15,8 +14,8 @@ async_client = AsyncOpenAI()
print('Running OpenAI v1.0.0+')
-# Assuming that instantiating agentops.Client will trigger the LlmTracker to override methods
-ao_client = agentops.Client(tags=['mock agent', openai.__version__])
+# Assuming that initializing will trigger the LlmTracker to override methods
+agentops.init(tags=['mock agent', openai.__version__])
# Now the client.chat.completions.create should be the overridden method
@@ -94,4 +93,4 @@ asyncio.run(test_async_chat_completion())
print('Running async streaming test')
asyncio.run(test_async_chat_completion_stream())
-ao_client.end_session('Success')
+agentops.end_session('Success')
diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_async.py b/tests/openai_handlers/_test_llm_tracker_ge_1_async.py
similarity index 100%
rename from tests/openai_handlers/test_llm_tracker_ge_1_async.py
rename to tests/openai_handlers/_test_llm_tracker_ge_1_async.py
diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/_test_llm_tracker_ge_1_sync.py
similarity index 100%
rename from tests/openai_handlers/test_llm_tracker_ge_1_sync.py
rename to tests/openai_handlers/_test_llm_tracker_ge_1_sync.py
diff --git a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/_test_llm_tracker_lt_1_sync.py
similarity index 100%
rename from tests/openai_handlers/test_llm_tracker_lt_1_sync.py
rename to tests/openai_handlers/_test_llm_tracker_lt_1_sync.py
diff --git a/tests/test_canary.py b/tests/test_canary.py
index ddceba50..48a37b13 100644
--- a/tests/test_canary.py
+++ b/tests/test_canary.py
@@ -1,8 +1,8 @@
import pytest
import requests_mock
import time
-
-from agentops import Client, Event
+import agentops
+from agentops import ActionEvent
@pytest.fixture
@@ -18,17 +18,17 @@ class TestCanary:
def setup_method(self):
self.url = 'https://api.agentops.ai'
self.api_key = "random_api_key"
- self.client = Client(api_key=self.api_key, max_wait_time=5)
+ agentops.init(api_key=self.api_key, max_wait_time=5)
def teardown_method(self):
- self.client.end_session(end_state='Success')
+ agentops.end_session(end_state='Success')
def test_agent_ops_record(self, mock_req):
# Arrange
event_type = 'test_event_type'
# Act
- self.client.record(Event(event_type))
+ agentops.record(ActionEvent(event_type))
time.sleep(0.1)
# Assert
diff --git a/tests/test_logger.py b/tests/test_logger.py
deleted file mode 100755
index 09b1835e..00000000
--- a/tests/test_logger.py
+++ /dev/null
@@ -1,89 +0,0 @@
-import pytest
-import requests_mock
-import time
-from agentops import Client, AgentOpsLogger
-
-
-def mock_req():
- with requests_mock.Mocker() as m:
- url = 'https://api.agentops.ai'
- m.post(url + '/events', text='ok')
- m.post(url + '/sessions', text='ok')
- yield m
-
-
-class TestLogger:
- def setup_method(self):
- self.url = 'https://api.agentops.ai'
- self.api_key = "random_api_key"
- self.client = Client(api_key=self.api_key, max_wait_time=5)
-
- def teardown_method(self):
- self.client.end_session(end_state='Success')
-
- def test_info(self, mock_req):
- # Arrange
- self.event_type = "info"
- self.logger = AgentOpsLogger.get_agentops_logger(
- self.client, self.event_type)
- test_message = "Test info message"
-
- # Act
- try:
- self.logger.info(test_message)
- except Exception as e:
- pytest.fail(f"test_info failed with {e}")
-
- time.sleep(0.1)
-
- # Assert
- assert len(mock_req.request_history) == 1
- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
- request_json = mock_req.last_request.json()
- assert request_json['events'][0]['event_type'] == f"{self.event_type}:INFO"
- assert request_json['events'][0]['returns'] == test_message
-
- def test_error(self, mock_req):
- # Arrange
- test_message = "Test error message"
- self.event_type = "error"
- self.logger = AgentOpsLogger.get_agentops_logger(
- self.client, self.event_type)
-
- # Act
- try:
- self.logger.error(test_message)
- except Exception as e:
- pytest.fail(f"test_error failed with {e}")
-
- time.sleep(0.1)
-
- # Assert
- assert len(mock_req.request_history) == 1
- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
- request_json = mock_req.last_request.json()
- assert request_json['events'][0]['event_type'] == f"{self.event_type}:ERROR"
- assert request_json['events'][0]['returns'] == test_message
-
- def test_warn(self, mock_req):
- # Arrange
- test_message = "Test warn message"
- self.event_type = "warning"
- self.logger = AgentOpsLogger.get_agentops_logger(
- self.client, self.event_type)
-
- # Act
- try:
- self.logger.warning(test_message)
- except Exception as e:
- pytest.fail(f"test_warn failed with {e}")
-
- time.sleep(0.1)
-
- # Assert
- assert len(mock_req.request_history) == 1
- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
- request_json = mock_req.last_request.json()
- assert request_json['events'][0]['event_type'] == f"{self.event_type}:WARNING"
- assert request_json['events'][0]['returns'] == test_message
diff --git a/tests/test_record_function.py b/tests/test_record_function.py
new file mode 100644
index 00000000..de03a81f
--- /dev/null
+++ b/tests/test_record_function.py
@@ -0,0 +1,90 @@
+import pytest
+import requests_mock
+import time
+import agentops
+from agentops import record_function
+from datetime import datetime
+
+def mock_req():
+ with requests_mock.Mocker() as m:
+ url = 'https://api.agentops.ai'
+ m.post(url + '/events', text='ok')
+ m.post(url + '/sessions', text='ok')
+ yield m
+
+class TestRecordAction:
+ def setup_method(self):
+ self.url = 'https://api.agentops.ai'
+ self.api_key = "random_api_key"
+ self.event_type = 'test_event_type'
+ agentops.init(self.api_key, max_wait_time=5, auto_start_session=False)
+ agentops.start_session()
+
+ def teardown_method(self):
+ agentops.end_session(end_state='Success')
+
+ def test_record_function_decorator(self, mock_req):
+ @record_function(event_name=self.event_type)
+ def add_two(x, y):
+ return x + y
+
+ # Act
+ add_two(3, 4)
+ time.sleep(0.1)
+
+ # Assert
+ assert len(mock_req.request_history) == 1
+ assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
+ request_json = mock_req.last_request.json()
+ assert request_json['events'][0]['action_type'] == self.event_type
+ assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}
+ assert request_json['events'][0]['returns'] == 7
+
+ def test_record_function_decorator_multiple(self, mock_req):
+ # Arrange
+ @record_function(event_name=self.event_type)
+ def add_three(x, y, z=3):
+ return x + y + z
+
+ # Act
+ add_three(1, 2)
+ time.sleep(0.1)
+ add_three(1, 2)
+ time.sleep(0.1)
+
+ # Assert
+ assert len(mock_req.request_history) == 2
+ assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
+ request_json = mock_req.last_request.json()
+ assert request_json['events'][0]['action_type'] == self.event_type
+ assert request_json['events'][0]['params'] == {'x': 1, 'y': 2, 'z': 3}
+ assert request_json['events'][0]['returns'] == 6
+
+ @pytest.mark.asyncio
+ async def test_async_function_call(self, mock_req):
+
+ @record_function(self.event_type)
+ async def async_add(x, y):
+ time.sleep(0.1)
+ return x + y
+
+ # Act
+ result = await async_add(3, 4)
+ time.sleep(0.1)
+
+ # Assert
+ assert result == 7
+ # Assert
+ assert len(mock_req.request_history) == 1
+ assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
+ request_json = mock_req.last_request.json()
+ assert request_json['events'][0]['action_type'] == self.event_type
+ assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}
+ assert request_json['events'][0]['returns'] == 7
+ init = datetime.fromisoformat(
+ request_json['events'][0]['init_timestamp'].replace('Z', '+00:00'))
+ end = datetime.fromisoformat(
+ request_json['events'][0]['end_timestamp'].replace('Z', '+00:00'))
+
+ assert (end - init).total_seconds() >= 0.1
\ No newline at end of file
diff --git a/tests/test_session.py b/tests/test_session.py
index 128b9c49..b61e715f 100644
--- a/tests/test_session.py
+++ b/tests/test_session.py
@@ -1,10 +1,8 @@
import pytest
import requests_mock
import time
-from datetime import datetime
-
-
-from agentops import Client, Event, record_function
+import agentops
+from agentops import ActionEvent
@pytest.fixture
@@ -20,19 +18,20 @@ class TestSessions:
def setup_method(self):
self.api_key = "random_api_key"
self.event_type = 'test_event_type'
+ agentops.init(api_key=self.api_key, max_wait_time=5, auto_start_session=False)
def test_session(self, mock_req):
# Arrange
- client = Client(api_key=self.api_key, max_wait_time=5)
+ agentops.start_session()
# Act
- client.record(Event(self.event_type))
+ agentops.record(ActionEvent(self.event_type))
# Assert the session has been initiated and the id has been created on backend.
assert len(mock_req.request_history) == 1
# Act
- client.record(Event(self.event_type))
+ agentops.record(ActionEvent(self.event_type))
time.sleep(0.1)
# Assert an event has been added
@@ -43,24 +42,23 @@ class TestSessions:
# Act
end_state = 'Success'
- client.end_session(end_state)
+ agentops.end_session(end_state)
time.sleep(0.1)
# Since a session has ended, no more events should be recorded, but end_session should be called
assert len(mock_req.request_history) == 3
assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
request_json = mock_req.last_request.json()
- assert request_json['session']['rating'] == None
assert request_json['session']['end_state'] == end_state
assert request_json['session']['tags'] == None
def test_tags(self, mock_req):
# Arrange
tags = ['GPT-4']
- client = Client(api_key=self.api_key, tags=tags, max_wait_time=5)
+ agentops.start_session(tags=tags)
# Act
- client.record(Event(self.event_type))
+ agentops.record(ActionEvent(self.event_type))
time.sleep(0.1)
# Assert 2 requests - 1 for session init, 1 for event
@@ -71,131 +69,12 @@ class TestSessions:
# Act
end_state = 'Success'
- client.end_session(end_state)
+ agentops.end_session(end_state)
time.sleep(0.1)
# Assert 3 requets, 1 for session init, 1 for event, 1 for end session
assert len(mock_req.request_history) == 3
assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
request_json = mock_req.last_request.json()
- assert request_json['session']['rating'] == None
assert request_json['session']['end_state'] == end_state
- assert request_json['session']['tags'] == tags
-
-
-class TestRecordAction:
- def setup_method(self):
- self.url = 'https://api.agentops.ai'
- self.api_key = "random_api_key"
- self.event_type = 'test_event_type'
- self.client = Client(self.api_key, max_wait_time=5)
-
- def teardown_method(self):
- self.client.end_session(end_state='Success')
-
- def test_record_function_decorator(self, mock_req):
- @record_function(event_name=self.event_type)
- def add_two(x, y):
- return x + y
-
- # Act
- add_two(3, 4)
- time.sleep(0.1)
-
- # Assert
- assert len(mock_req.request_history) == 1
- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
- request_json = mock_req.last_request.json()
- assert request_json['events'][0]['event_type'] == self.event_type
- assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}
- assert request_json['events'][0]['returns'] == 7
- assert request_json['events'][0]['result'] == 'Success'
- assert request_json['events'][0]['tags'] == ['foo', 'bar']
-
- def test_record_function_decorator_multiple(self, mock_req):
- # Arrange
- @record_function(event_name=self.event_type)
- def add_three(x, y, z=3):
- return x + y + z
-
- # Act
- add_three(1, 2)
- time.sleep(0.1)
-
- # Assert
- assert len(mock_req.request_history) == 1
- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
- request_json = mock_req.last_request.json()
- assert request_json['events'][0]['event_type'] == self.event_type
- assert request_json['events'][0]['params'] == {'x': 1, 'y': 2, 'z': 3}
- assert request_json['events'][0]['returns'] == 6
- assert request_json['events'][0]['result'] == 'Success'
- assert request_json['events'][0]['tags'] == ['foo', 'bar']
-
- @pytest.mark.asyncio
- async def test_async_function_call(self, mock_req):
-
- @record_function(self.event_type)
- async def async_add(x, y):
- time.sleep(0.1)
- return x + y
-
- # Act
- result = await async_add(3, 4)
- time.sleep(0.1)
-
- # Assert
- assert result == 7
- # Assert
- assert len(mock_req.request_history) == 1
- assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_key
- request_json = mock_req.last_request.json()
- assert request_json['events'][0]['event_type'] == self.event_type
- assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}
- assert request_json['events'][0]['returns'] == 7
- assert request_json['events'][0]['result'] == 'Success'
- init = datetime.fromisoformat(
- request_json['events'][0]['init_timestamp'].replace('Z', '+00:00'))
- end = datetime.fromisoformat(
- request_json['events'][0]['end_timestamp'].replace('Z', '+00:00'))
-
- assert (end - init).total_seconds() >= 0.1
-
- def test_function_call(self, mock_req):
- # Arrange
- prompt = 'prompt'
-
- @record_function(event_name=self.event_type)
- def foo(prompt=prompt):
- return 'output'
-
- # Act
- foo()
- time.sleep(0.1)
-
- # Assert
- assert len(mock_req.request_history) == 1
- request_json = mock_req.last_request.json()
- assert request_json['events'][0]['action_type'] == 'action'
- assert request_json['events'][0]['prompt'] == None
- assert request_json['events'][0]['returns'] == 'output'
- assert request_json['events'][0]['result'] == 'Success'
-
- def test_llm_call_no_action_type(self, mock_req):
- # Arrange
- prompt = 'prompt'
-
- @record_function(event_name=self.event_type)
- def llm_call(prompt=prompt):
- return 'output'
-
- llm_call()
- time.sleep(0.1)
-
- # Assert
- assert len(mock_req.request_history) == 1
- request_json = mock_req.last_request.json()
- assert request_json['events'][0]['action_type'] == 'action'
- assert request_json['events'][0]['prompt'] == None
- assert request_json['events'][0]['returns'] == 'output'
- assert request_json['events'][0]['result'] == 'Success'
+ assert request_json['session']['tags'] == tags
\ No newline at end of file
diff --git a/tox.ini b/tox.ini
index 8f93df07..86213961 100644
--- a/tox.ini
+++ b/tox.ini
@@ -17,6 +17,7 @@ deps =
mypy: mypy
types-requests
psutil
+ openai
commands =
coverage run --source agentops -m pytest
coverage report -m