Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
1 create enum for session end states and event results (#12)
Alex Reibman2fde08322b92
diff --git a/agentops/agentops.py b/agentops/agentops.pyindex f0d39d25..cc194aed 100644--- a/agentops/agentops.py+++ b/agentops/agentops.py@@ -6,10 +6,9 @@ Classes:"""from .config import Configuration-from .event import Session, Event+from .event import Session, Event, EventStatefrom .worker import Workerfrom uuid import uuid4-import jsonfrom typing import Optional, Dictimport functoolsimport inspect@@ -63,29 +62,28 @@ class AgentOps:arg_names = list(func_args.keys())arg_values = dict(zip(arg_names, args))arg_values.update(kwargs)-try:- output = func(*args, **kwargs)+ returns = func(*args, **kwargs)# Record the event after the function callself.record(Event(event_type=event_name,params=arg_values,- output=output,- result="SUCCESS",+ returns=returns,+ result="Success",tags=tags))except Exception as e:# Record the event after the function callself.record(Event(event_type=event_name,params=arg_values,- output=None,- result='FAIL',+ returns=None,+ result='Fail',tags=tags))# Re-raise the exceptionraise- return output+ return returnsreturn wrapper@@ -102,7 +100,7 @@ class AgentOps:self.worker = Worker(self.config)self.worker.start_session(self.session)- def end_session(self, end_state: Optional[str] = None, rating: Optional[str] = None):+ def end_session(self, end_state: EventState = EventState.INDETERMINATE, rating: Optional[str] = None):"""End the current session with the AgentOps service.@@ -110,5 +108,10 @@ class AgentOps:end_state (str, optional): The final state of the session.rating (str, optional): The rating for the session."""+ valid_results = set(vars(EventState).values())+ if end_state not in valid_results:+ raise ValueError(+ f"end_state must be one of {EventState.__args__}. Provided: {end_state}")+self.session.end_session(end_state, rating)self.worker.end_session(self.session)diff --git a/agentops/event.py b/agentops/event.pyindex 135d5d2c..bc4cd623 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -1,6 +1,3 @@-from .helpers import get_ISO_time-from typing import Optional, Dict, Literal-"""AgentOps events.@@ -8,6 +5,20 @@ Classes:Event: Represents discrete events to be recorded.Session: Represents a session of events, with a start and end state."""+from .helpers import get_ISO_time+from typing import Optional, Dict+++class EventState:+ SUCCESS = "Success"+ FAIL = "Fail"+ INDETERMINATE = "Indeterminate"+++class SessionState:+ SUCCESS = "Success"+ FAIL = "Fail"+ INDETERMINATE = "Indeterminate"class Event:@@ -17,8 +28,8 @@ class Event:Args:event_type (str): Type of the event, e.g., "API Call". Required.params (str, optional): The parameters passed to the operation.- output (str, optional): The output of the operation.- result (str, optional): Result of the operation, e.g., "success", "fail", "indeterminate".+ returns (str, optional): The output of the operation.+ result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate".tags (Dict[str, str], optional): Tags that can be used for grouping or sorting later. e.g. {"llm": "GPT-4"}.@@ -28,13 +39,13 @@ class Event:def __init__(self, event_type: str,params: Optional[str] = None,- output: Optional[str] = None,- result: Optional[str] = None,+ returns: Optional[str] = None,+ result: EventState = EventState.INDETERMINATE,tags: Optional[Dict[str, str]] = None):self.event_type = event_typeself.params = params- self.output = output+ self.returns = returnsself.result = resultself.tags = tagsself.timestamp = get_ISO_time()@@ -61,7 +72,7 @@ class Session:self.init_timestamp = get_ISO_time()self.tags = tags- def end_session(self, end_state: Optional[str], rating: Optional[str] = None):+ def end_session(self, end_state: SessionState = SessionState.INDETERMINATE, rating: Optional[str] = None):"""End the session with a specified state and rating.@@ -69,6 +80,10 @@ class Session:end_state (str, optional): The final state of the session. Suggested: "Success", "Fail", "Indeterminate"rating (str, optional): The rating for the session."""+ valid_results = set(vars(SessionState).values())+ if end_state not in valid_results:+ raise ValueError(+ f"end_state must be one of {valid_results}. Provided: {end_state}")self.end_state = end_stateself.rating = ratingself.end_timestamp = get_ISO_time()diff --git a/agentops/http.py b/agentops/http.pyindex 70cf0ed8..c8abda75 100644--- a/agentops/http.py+++ b/agentops/http.py@@ -69,7 +69,7 @@ class HttpClient:JSON_HEADER["X-Agentops-Auth"] = api_keyres = request_session.post(url, data=payload,- headers=JSON_HEADER, timeout=20)+ headers=JSON_HEADER, timeout=20)result.parse(res)except requests.exceptions.Timeout:diff --git a/agentops/logger.py b/agentops/logger.pyindex 43f9fae1..2f71cbb5 100755--- a/agentops/logger.py+++ b/agentops/logger.py@@ -1,26 +1,27 @@import loggingimport refrom .agentops import AgentOps-from .event import Event+from .event import Event, EventState+class AgentOpsLogger():"""A utility class for creating loggers and handlers configured to work with the AgentOps service.-+This class provides two static methods for creating a logger or a handler that sends logrecords to the AgentOps service. The logger and handler are configured with a specificAgentOps client and name.-+Example Usage:-+>>> from agentops import AgentOps>>> client = AgentOps(...)>>> logger = AgentOpsLogger.get_agentops_logger(client, 'my_logger')>>> logger.info('This is an info log')-+This will send an 'info' log to the AgentOps service."""-+@staticmethoddef get_agentops_logger(client: AgentOps, name: str, level=logging.DEBUG):"""@@ -40,7 +41,7 @@ class AgentOpsLogger():handler.setLevel(level)logger.addHandler(handler)return logger-+@staticmethoddef get_agentops_handler(client: AgentOps, name: str):"""@@ -103,9 +104,9 @@ class AgentOpsHandler(logging.Handler):log_entry = self.remove_color_codes(log_entry)if record.levelno == logging.ERROR:- result = "fail"+ result = EventState.FAILelse:- result = "indeterminate"+ result = EventState.INDETERMINATEself.client.record(- Event(f'{self.name}:{record.levelname}', output=log_entry, result=result))+ Event(f'{self.name}:{record.levelname}', returns=log_entry, result=result))diff --git a/tests/test_canary.py b/tests/test_canary.pyindex 5f268eca..a6fe44e0 100644--- a/tests/test_canary.py+++ b/tests/test_canary.py@@ -3,6 +3,7 @@ import requests_mockimport timefrom agentops import AgentOps, Event, Configuration+from agentops.event import EventState@pytest.fixture@@ -13,6 +14,7 @@ def mock_req():m.post(url + '/sessions', text='ok')yield m+class TestCanary:def setup_method(self):self.url = 'https://agentops-server-v2.fly.dev'@@ -21,7 +23,7 @@ class TestCanary:self.client = AgentOps(api_key=self.api_key, config=self.config)def teardown_method(self):- self.client.end_session(end_state="success")+ self.client.end_session(end_state=EventState.SUCCESS)def test_agent_ops_record(self, mock_req):# Arrangediff --git a/tests/test_logger.py b/tests/test_logger.pyindex 0b04b690..7b1c0355 100755--- a/tests/test_logger.py+++ b/tests/test_logger.py@@ -2,6 +2,7 @@ import pytestimport requests_mockimport timefrom agentops import AgentOps, AgentOpsLogger, Configuration+from agentops.event import EventState@pytest.fixture@@ -20,9 +21,8 @@ class TestLogger:self.config = Configuration(max_wait_time=5)self.client = AgentOps(api_key=self.api_key, config=self.config)-def teardown_method(self):- self.client.end_session(end_state="success")+ self.client.end_session(end_state=EventState.SUCCESS)def test_info(self, mock_req):# Arrange@@ -37,7 +37,6 @@ class TestLogger:except Exception as e:pytest.fail(f"test_info failed with {e}")-time.sleep(0.1)# Assert@@ -45,7 +44,7 @@ class TestLogger:assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json['events'][0]['event_type'] == f"{self.event_type}:INFO"- assert request_json['events'][0]['output'] == test_message+ assert request_json['events'][0]['returns'] == test_messagedef test_error(self, mock_req):# Arrange@@ -67,7 +66,7 @@ class TestLogger:assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json['events'][0]['event_type'] == f"{self.event_type}:ERROR"- assert request_json['events'][0]['output'] == test_message+ assert request_json['events'][0]['returns'] == test_messagedef test_warn(self, mock_req):# Arrange@@ -89,4 +88,4 @@ class TestLogger:assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json['events'][0]['event_type'] == f"{self.event_type}:WARNING"- assert request_json['events'][0]['output'] == test_message+ assert request_json['events'][0]['returns'] == test_messagediff --git a/tests/test_session.py b/tests/test_session.pyindex eaa9ff19..b1a86e29 100644--- a/tests/test_session.py+++ b/tests/test_session.py@@ -3,6 +3,7 @@ import requests_mockimport timefrom agentops import AgentOps, Event, Configuration+from agentops.event import EventState@pytest.fixture@@ -27,25 +28,26 @@ class TestSessions:# Actclient.record(Event(self.event_type))- # Assert- assert len(mock_req.request_history) == 0+ # Assert the session has been initiated and the id has been created on backend.+ assert len(mock_req.request_history) == 1# Actclient.record(Event(self.event_type))time.sleep(0.1)- # Assert- assert len(mock_req.request_history) == 1+ # Assert an event has been added+ assert len(mock_req.request_history) == 2assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json['events'][0]['event_type'] == self.event_type# Act- end_state = "Succeed"+ end_state = EventState.SUCCESSclient.end_session(end_state)time.sleep(0.1)- assert len(mock_req.request_history) == 2+ # Since a session has ended, no more events should be recorded, but end_session should be called+ assert len(mock_req.request_history) == 3assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json['session']['rating'] == None@@ -62,22 +64,23 @@ class TestSessions:client.record(Event(self.event_type))time.sleep(0.1)- # Assert- assert len(mock_req.request_history) == 1+ # Assert 2 requests - 1 for session init, 1 for event+ assert len(mock_req.request_history) == 2assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json['events'][0]['event_type'] == self.event_type# Act- end_state = "Succeed"- client.end_session()+ end_state = EventState.SUCCESS+ client.end_session(end_state)time.sleep(0.1)- assert len(mock_req.request_history) == 2+ # Assert 3 requets, 1 for session init, 1 for event, 1 for end session+ assert len(mock_req.request_history) == 3assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json['session']['rating'] == None- assert request_json['session']['end_state'] == None+ assert request_json['session']['end_state'] == end_stateassert request_json['session']['tags'] == tags@@ -90,7 +93,7 @@ class TestRecordAction:self.client = AgentOps(self.api_key, config=self.config)def teardown_method(self):- self.client.end_session(end_state="success")+ self.client.end_session(end_state=EventState.SUCCESS)def test_record_action_decorator(self, mock_req):@self.client.record_action(event_name=self.event_type, tags={'foo': 'bar'})@@ -107,8 +110,8 @@ class TestRecordAction:assert request_json['event']['event_type'] == self.event_typeassert request_json['event']['params'] == {'args': [3, 4], 'kwargs': {}}- assert request_json['event']['output'] == 7- assert request_json['event']['result'] == 'SUCCESS'+ assert request_json['event']['returns'] == 7+ assert request_json['event']['result'] == EventState.SUCCESSassert request_json['event']['tags'] == {'foo': 'bar'}def test_record_action_decorator(self, mock_req):@@ -127,6 +130,6 @@ class TestRecordAction:request_json = mock_req.last_request.json()assert request_json['events'][0]['event_type'] == self.event_typeassert request_json['events'][0]['params'] == {'x': 3, 'y': 4}- assert request_json['events'][0]['output'] == 7- assert request_json['events'][0]['result'] == 'SUCCESS'+ assert request_json['events'][0]['returns'] == 7+ assert request_json['events'][0]['result'] == EventState.SUCCESSassert request_json['events'][0]['tags'] == {'foo': 'bar'}