Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
28 update events to have init + end timestamps (#29)
Alex Reibman9142d83af7ac
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 55aaf02b..99bdd366 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -1,5 +1,4 @@from .client import Client-from .event import Event, EventState-from .session import SessionState+from .event import Eventfrom .logger import AgentOpsLoggerfrom .helpers import Models, ActionTypediff --git a/agentops/client.py b/agentops/client.pyindex 9695fdcb..cd16284b 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -5,12 +5,13 @@ Classes:Client: Provides methods to interact with the AgentOps service."""-from .event import Event, EventState-from .helpers import Models, ActionType-from .session import Session, SessionState+from .event import Event+from .helpers import Models, ActionType, get_ISO_time+from .session import Sessionfrom .worker import Workerfrom uuid import uuid4from typing import Optional, List+from pydantic import Fieldimport functoolsimport loggingimport inspect@@ -81,12 +82,10 @@ class Client:frame: The current stack frame."""logging.info('Signal SIGTERM or SIGINT detected. Ending session...')- self.end_session(end_state=EventState.FAIL)+ self.end_session(end_state='Fail')sys.exit(0)- def record(self, event: Event,- action_type: ActionType = ActionType.ACTION,- model: Optional[Models] = None):+ def record(self, event: Event):"""Record an event with the AgentOps service.@@ -150,6 +149,7 @@ class Client:def decorator(func):@functools.wraps(func)def wrapper(*args, **kwargs):+ init_time = get_ISO_time()func_args = inspect.signature(func).parametersarg_names = list(func_args.keys())# Get default values@@ -190,10 +190,11 @@ class Client:self.record(Event(event_type=event_name,params=arg_values,returns=returns,- result=EventState.SUCCESS,+ result='Success',action_type=action,model=model,prompt=prompt,+ init_timestamp=init_time,tags=tags))except Exception as e:@@ -201,10 +202,11 @@ class Client:self.record(Event(event_type=event_name,params=arg_values,returns=None,- result=EventState.FAIL,+ result='Fail',action_type=action,model=model,prompt=prompt,+ init_timestamp=init_time,tags=tags))# Re-raise the exception@@ -228,7 +230,9 @@ class Client:self.worker = Worker(self.config)self.worker.start_session(self.session)- def end_session(self, end_state: SessionState = SessionState.INDETERMINATE,+ def end_session(self, end_state: str = Field("Indeterminate",+ description="End state of the session",+ pattern="^(Success|Fail|Indeterminate)$"),rating: Optional[str] = None):"""End the current session with the AgentOps service.@@ -237,10 +241,6 @@ class Client:end_state (str, optional): The final state of the session.rating (str, optional): The rating for the session."""- valid_results = set(vars(SessionState).values())- if end_state not in valid_results:- raise ValueError(- f"end_state must be one of {SessionState}. Provided: {end_state}")if not self.session.has_ended:self.session.end_session(end_state, rating)self.worker.end_session(self.session)@@ -250,4 +250,4 @@ class Client:def cleanup(self):# Only run cleanup function if session is createdif hasattr(self, "session") and not self.session.has_ended:- self.end_session(end_state=SessionState.FAIL)+ self.end_session(end_state='Fail')diff --git a/agentops/event.py b/agentops/event.pyindex a56f91d3..e069df91 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -6,12 +6,7 @@ Classes:"""from .helpers import get_ISO_time, ActionType, Modelsfrom typing import Optional, List---class EventState:- SUCCESS = "Success"- FAIL = "Fail"- INDETERMINATE = "Indeterminate"+from pydantic import Fieldclass Event:@@ -22,28 +17,41 @@ class Event:event_type (str): Type of the event, e.g., "API Call". Required.params (str, optional): The parameters passed to the operation.returns (str, optional): The output of the operation.- result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate".- action_type (ActionType, optional): Type of action of the evnet e.g. 'action', 'llm', 'api'+ result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate". Defaults to "Indeterminate".+ action_type (ActionType, optional): Type of action of the event e.g. 'action', 'llm', 'api'. Defaults to ActionType.ACTION.model (Models, optional): The model used during the event if an LLM is used (i.e. GPT-4).For models, see the types available in the Models enum.If a model is set but an action_type is not, the action_type will be coerced to 'llm'.Defaults to None.- prompt (str, optional): The input prompt for an LLM call when an LLM is being used.- tags (List[str], optional): Tags that can be used for grouping or sorting later. e.g. ["my_tag"].-+ prompt (str, optional): The input prompt for an LLM call when an LLM is being used. Defaults to None.+ tags (List[str], optional): Tags that can be used for grouping or sorting later. e.g. ["my_tag"]. Defaults to None.+ init_timestamp (float, optional): The timestamp for when the event was initiated, represented as seconds since the epoch.+ Defaults to the end timestamp.Attributes:- timestamp (float): The timestamp for when the event was created, represented as seconds since the epoch.+ event_type (str): Type of the event.+ params (str, optional): The parameters passed to the operation.+ returns (str, optional): The output of the operation.+ result (str): Result of the operation.+ action_type (ActionType): Type of action of the event.+ model (Models, optional): The model used during the event.+ prompt (str, optional): The input prompt for an LLM call.+ tags (List[str], optional): Tags associated with the event.+ end_timestamp (float): The timestamp for when the event ended, represented as seconds since the epoch.+ init_timestamp (float): The timestamp for when the event was initiated, represented as seconds since the epoch."""def __init__(self, event_type: str,params: Optional[str] = None,returns: Optional[str] = None,- result: EventState = EventState.INDETERMINATE,+ result: str = Field("Indeterminate",+ description="Result of the operation",+ pattern="^(Success|Fail|Indeterminate)$"),action_type: Optional[ActionType] = ActionType.ACTION,model: Optional[Models] = None,prompt: Optional[str] = None,- tags: Optional[List[str]] = None+ tags: Optional[List[str]] = None,+ init_timestamp: Optional[float] = None):self.event_type = event_typeself.params = params@@ -53,4 +61,5 @@ class Event:self.action_type = action_typeself.model = modelself.prompt = prompt- self.timestamp = get_ISO_time()+ self.end_timestamp = get_ISO_time()+ self.init_timestamp = init_timestamp if init_timestamp else self.end_timestampdiff --git a/agentops/logger.py b/agentops/logger.pyindex be2166fe..dd472377 100755--- a/agentops/logger.py+++ b/agentops/logger.py@@ -1,7 +1,7 @@import loggingimport refrom .client import Client-from .event import Event, EventState+from .event import Eventclass AgentOpsLogger():@@ -104,9 +104,9 @@ class AgentOpsHandler(logging.Handler):log_entry = self.remove_color_codes(log_entry)if record.levelno == logging.ERROR:- result = EventState.FAIL+ result = "Fail"else:- result = EventState.INDETERMINATE+ result = 'Indeterminate'self.client.record(Event(f'{self.name}:{record.levelname}', returns=log_entry, result=result))diff --git a/agentops/session.py b/agentops/session.pyindex ade8c945..e29c6cad 100644--- a/agentops/session.py+++ b/agentops/session.py@@ -1,11 +1,10 @@from .helpers import get_ISO_time+from pydantic import BaseModel, Fieldfrom typing import Optional, List-class SessionState:- SUCCESS = "Success"- FAIL = "Fail"- INDETERMINATE = "Indeterminate"+class SessionState(BaseModel):+ end_state: str = Field(..., pattern="^(Success|Fail|Indeterminate)$")class Session:@@ -29,18 +28,15 @@ class Session:self.init_timestamp = get_ISO_time()self.tags = tags- def end_session(self, end_state: SessionState = SessionState.INDETERMINATE, rating: Optional[str] = None):+ def end_session(self, end_state: str = "Indeterminate", rating: Optional[str] = None):"""End the session with a specified state and rating.Args:- end_state (str, optional): The final state of the session. Suggested: "Success", "Fail", "Indeterminate"+ end_state (str, optional): The final state of the session. Options: "Success", "Fail", "Indeterminate"rating (str, optional): The rating for the session."""- valid_results = set(vars(SessionState).values())- if end_state not in valid_results:- raise ValueError(- f"end_state must be one of {valid_results}. Provided: {end_state}")+ SessionState(end_state=end_state)self.end_state = end_stateself.rating = ratingself.end_timestamp = get_ISO_time()diff --git a/requirements.txt b/requirements.txtindex dd5a435b..45f6e9f6 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,3 +1,4 @@pytest==7.4.0requests==2.31.0-requests-mock==1.11.0\ No newline at end of file+requests-mock==1.11.0+pydantic==2.4.2\ No newline at end of filediff --git a/tests/mock_agent.py b/tests/mock_agent.pyindex d8713d15..b25a7186 100644--- a/tests/mock_agent.py+++ b/tests/mock_agent.py@@ -3,14 +3,27 @@ import timeprint('init')-ao_client = agentops.Client(api_key='floof', tags=['mock tests'])+ao_client = agentops.Client(api_key='floof',+ tags=['mock tests'])+++@ao_client.record_action('action')+def sleep_func(sleep):+ time.sleep(sleep)+ print('1')+++@ao_client.record_action('multi')+def multi_event(sleep):+ sleep_func(1)+ time.sleep(sleep)print('Action 1')-ao_client.record_action('Action 1')-time.sleep(0.1)+# ao_client.record_action('Action 1')+sleep_func(0.1)print('Action 2 - 5 second sleep')-ao_client.record_action('Action 2')-time.sleep(5)+# ao_client.record_action('Action 2')+multi_event(3)print('End Session')ao_client.end_session(end_state='Fail')diff --git a/tests/test_canary.py b/tests/test_canary.pyindex d3d74fcb..60b0ebde 100644--- a/tests/test_canary.py+++ b/tests/test_canary.py@@ -2,7 +2,7 @@ import pytestimport requests_mockimport time-from agentops import Client, Event, SessionState+from agentops import Client, Event@pytest.fixture@@ -21,7 +21,7 @@ class TestCanary:self.client = Client(api_key=self.api_key, max_wait_time=5)def teardown_method(self):- self.client.end_session(end_state=SessionState.SUCCESS)+ self.client.end_session(end_state='Success')def test_agent_ops_record(self, mock_req):# Arrangediff --git a/tests/test_logger.py b/tests/test_logger.pyindex a8fe94a5..6ad66faf 100755--- a/tests/test_logger.py+++ b/tests/test_logger.py@@ -1,7 +1,7 @@import pytestimport requests_mockimport time-from agentops import Client, AgentOpsLogger, SessionState+from agentops import Client, AgentOpsLogger@pytest.fixture@@ -20,7 +20,7 @@ class TestLogger:self.client = Client(api_key=self.api_key, max_wait_time=5)def teardown_method(self):- self.client.end_session(end_state=SessionState.SUCCESS)+ self.client.end_session(end_state='Success')def test_info(self, mock_req):# Arrangediff --git a/tests/test_session.py b/tests/test_session.pyindex eeab3696..8ff26553 100644--- a/tests/test_session.py+++ b/tests/test_session.py@@ -2,7 +2,7 @@ import pytestimport requests_mockimport time-from agentops import Client, Event, EventState+from agentops import Client, Event@pytest.fixture@@ -40,7 +40,7 @@ class TestSessions:assert request_json['events'][0]['event_type'] == self.event_type# Act- end_state = EventState.SUCCESS+ end_state = 'Success'client.end_session(end_state)time.sleep(0.1)@@ -68,7 +68,7 @@ class TestSessions:assert request_json['events'][0]['event_type'] == self.event_type# Act- end_state = EventState.SUCCESS+ end_state = 'Success'client.end_session(end_state)time.sleep(0.1)@@ -89,7 +89,7 @@ class TestRecordAction:self.client = Client(self.api_key, max_wait_time=5)def teardown_method(self):- self.client.end_session(end_state=EventState.SUCCESS)+ self.client.end_session(end_state='Success')def test_record_action_decorator(self, mock_req):@self.client.record_action(event_name=self.event_type, tags=['foo', 'bar'])@@ -107,7 +107,7 @@ class TestRecordAction:assert request_json['events'][0]['event_type'] == self.event_typeassert request_json['events'][0]['params'] == {'x': 3, 'y': 4}assert request_json['events'][0]['returns'] == 7- assert request_json['events'][0]['result'] == EventState.SUCCESS+ assert request_json['events'][0]['result'] == 'Success'assert request_json['events'][0]['tags'] == ['foo', 'bar']def test_record_action_decorator_multiple(self, mock_req):@@ -127,7 +127,7 @@ class TestRecordAction:assert request_json['events'][0]['event_type'] == self.event_typeassert request_json['events'][0]['params'] == {'x': 1, 'y': 2, 'z': 3}assert request_json['events'][0]['returns'] == 6- assert request_json['events'][0]['result'] == EventState.SUCCESS+ assert request_json['events'][0]['result'] == 'Success'assert request_json['events'][0]['tags'] == ['foo', 'bar']def test_llm_call(self, mock_req):@@ -148,7 +148,7 @@ class TestRecordAction:assert request_json['events'][0]['action_type'] == 'llm'assert request_json['events'][0]['prompt'] == promptassert request_json['events'][0]['returns'] == 'output'- assert request_json['events'][0]['result'] == EventState.SUCCESS+ assert request_json['events'][0]['result'] == 'Success'def test_llm_call_no_prompt(self, mock_req):# Arrange@@ -190,4 +190,4 @@ class TestRecordAction:assert request_json['events'][0]['action_type'] == 'llm'assert request_json['events'][0]['prompt'] == promptassert request_json['events'][0]['returns'] == 'output'- assert request_json['events'][0]['result'] == EventState.SUCCESS+ assert request_json['events'][0]['result'] == 'Success'