Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

28 update events to have init + end timestamps (#29)

Alex Reibman9142d83af7ac
diff --git a/agentops/__init__.py b/agentops/__init__.py
index 55aaf02b..99bdd366 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -1,5 +1,4 @@
from .client import Client
-from .event import Event, EventState
-from .session import SessionState
+from .event import Event
from .logger import AgentOpsLogger
from .helpers import Models, ActionType
diff --git a/agentops/client.py b/agentops/client.py
index 9695fdcb..cd16284b 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -5,12 +5,13 @@ Classes:
Client: Provides methods to interact with the AgentOps service.
"""
-from .event import Event, EventState
-from .helpers import Models, ActionType
-from .session import Session, SessionState
+from .event import Event
+from .helpers import Models, ActionType, get_ISO_time
+from .session import Session
from .worker import Worker
from uuid import uuid4
from typing import Optional, List
+from pydantic import Field
import functools
import logging
import inspect
@@ -81,12 +82,10 @@ class Client:
frame: The current stack frame.
"""
logging.info('Signal SIGTERM or SIGINT detected. Ending session...')
- self.end_session(end_state=EventState.FAIL)
+ self.end_session(end_state='Fail')
sys.exit(0)
- def record(self, event: Event,
- action_type: ActionType = ActionType.ACTION,
- model: Optional[Models] = None):
+ def record(self, event: Event):
"""
Record an event with the AgentOps service.
@@ -150,6 +149,7 @@ class Client:
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
+ init_time = get_ISO_time()
func_args = inspect.signature(func).parameters
arg_names = list(func_args.keys())
# Get default values
@@ -190,10 +190,11 @@ class Client:
self.record(Event(event_type=event_name,
params=arg_values,
returns=returns,
- result=EventState.SUCCESS,
+ result='Success',
action_type=action,
model=model,
prompt=prompt,
+ init_timestamp=init_time,
tags=tags))
except Exception as e:
@@ -201,10 +202,11 @@ class Client:
self.record(Event(event_type=event_name,
params=arg_values,
returns=None,
- result=EventState.FAIL,
+ result='Fail',
action_type=action,
model=model,
prompt=prompt,
+ init_timestamp=init_time,
tags=tags))
# Re-raise the exception
@@ -228,7 +230,9 @@ class Client:
self.worker = Worker(self.config)
self.worker.start_session(self.session)
- def end_session(self, end_state: SessionState = SessionState.INDETERMINATE,
+ def end_session(self, end_state: str = Field("Indeterminate",
+ description="End state of the session",
+ pattern="^(Success|Fail|Indeterminate)$"),
rating: Optional[str] = None):
"""
End the current session with the AgentOps service.
@@ -237,10 +241,6 @@ class Client:
end_state (str, optional): The final state of the session.
rating (str, optional): The rating for the session.
"""
- valid_results = set(vars(SessionState).values())
- if end_state not in valid_results:
- raise ValueError(
- f"end_state must be one of {SessionState}. Provided: {end_state}")
if not self.session.has_ended:
self.session.end_session(end_state, rating)
self.worker.end_session(self.session)
@@ -250,4 +250,4 @@ class Client:
def cleanup(self):
# Only run cleanup function if session is created
if hasattr(self, "session") and not self.session.has_ended:
- self.end_session(end_state=SessionState.FAIL)
+ self.end_session(end_state='Fail')
diff --git a/agentops/event.py b/agentops/event.py
index a56f91d3..e069df91 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -6,12 +6,7 @@ Classes:
"""
from .helpers import get_ISO_time, ActionType, Models
from typing import Optional, List
-
-
-class EventState:
- SUCCESS = "Success"
- FAIL = "Fail"
- INDETERMINATE = "Indeterminate"
+from pydantic import Field
class Event:
@@ -22,28 +17,41 @@ class Event:
event_type (str): Type of the event, e.g., "API Call". Required.
params (str, optional): The parameters passed to the operation.
returns (str, optional): The output of the operation.
- result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate".
- action_type (ActionType, optional): Type of action of the evnet e.g. 'action', 'llm', 'api'
+ result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate". Defaults to "Indeterminate".
+ action_type (ActionType, optional): Type of action of the event e.g. 'action', 'llm', 'api'. Defaults to ActionType.ACTION.
model (Models, optional): The model used during the event if an LLM is used (i.e. GPT-4).
For models, see the types available in the Models enum.
If a model is set but an action_type is not, the action_type will be coerced to 'llm'.
Defaults to None.
- prompt (str, optional): The input prompt for an LLM call when an LLM is being used.
- tags (List[str], optional): Tags that can be used for grouping or sorting later. e.g. ["my_tag"].
-
+ prompt (str, optional): The input prompt for an LLM call when an LLM is being used. Defaults to None.
+ tags (List[str], optional): Tags that can be used for grouping or sorting later. e.g. ["my_tag"]. Defaults to None.
+ init_timestamp (float, optional): The timestamp for when the event was initiated, represented as seconds since the epoch.
+ Defaults to the end timestamp.
Attributes:
- timestamp (float): The timestamp for when the event was created, represented as seconds since the epoch.
+ event_type (str): Type of the event.
+ params (str, optional): The parameters passed to the operation.
+ returns (str, optional): The output of the operation.
+ result (str): Result of the operation.
+ action_type (ActionType): Type of action of the event.
+ model (Models, optional): The model used during the event.
+ prompt (str, optional): The input prompt for an LLM call.
+ tags (List[str], optional): Tags associated with the event.
+ end_timestamp (float): The timestamp for when the event ended, represented as seconds since the epoch.
+ init_timestamp (float): The timestamp for when the event was initiated, represented as seconds since the epoch.
"""
def __init__(self, event_type: str,
params: Optional[str] = None,
returns: Optional[str] = None,
- result: EventState = EventState.INDETERMINATE,
+ result: str = Field("Indeterminate",
+ description="Result of the operation",
+ pattern="^(Success|Fail|Indeterminate)$"),
action_type: Optional[ActionType] = ActionType.ACTION,
model: Optional[Models] = None,
prompt: Optional[str] = None,
- tags: Optional[List[str]] = None
+ tags: Optional[List[str]] = None,
+ init_timestamp: Optional[float] = None
):
self.event_type = event_type
self.params = params
@@ -53,4 +61,5 @@ class Event:
self.action_type = action_type
self.model = model
self.prompt = prompt
- self.timestamp = get_ISO_time()
+ self.end_timestamp = get_ISO_time()
+ self.init_timestamp = init_timestamp if init_timestamp else self.end_timestamp
diff --git a/agentops/logger.py b/agentops/logger.py
index be2166fe..dd472377 100755
--- a/agentops/logger.py
+++ b/agentops/logger.py
@@ -1,7 +1,7 @@
import logging
import re
from .client import Client
-from .event import Event, EventState
+from .event import Event
class AgentOpsLogger():
@@ -104,9 +104,9 @@ class AgentOpsHandler(logging.Handler):
log_entry = self.remove_color_codes(log_entry)
if record.levelno == logging.ERROR:
- result = EventState.FAIL
+ result = "Fail"
else:
- result = EventState.INDETERMINATE
+ result = 'Indeterminate'
self.client.record(
Event(f'{self.name}:{record.levelname}', returns=log_entry, result=result))
diff --git a/agentops/session.py b/agentops/session.py
index ade8c945..e29c6cad 100644
--- a/agentops/session.py
+++ b/agentops/session.py
@@ -1,11 +1,10 @@
from .helpers import get_ISO_time
+from pydantic import BaseModel, Field
from typing import Optional, List
-class SessionState:
- SUCCESS = "Success"
- FAIL = "Fail"
- INDETERMINATE = "Indeterminate"
+class SessionState(BaseModel):
+ end_state: str = Field(..., pattern="^(Success|Fail|Indeterminate)$")
class Session:
@@ -29,18 +28,15 @@ class Session:
self.init_timestamp = get_ISO_time()
self.tags = tags
- def end_session(self, end_state: SessionState = SessionState.INDETERMINATE, rating: Optional[str] = None):
+ def end_session(self, end_state: str = "Indeterminate", rating: Optional[str] = None):
"""
End the session with a specified state and rating.
Args:
- end_state (str, optional): The final state of the session. Suggested: "Success", "Fail", "Indeterminate"
+ end_state (str, optional): The final state of the session. Options: "Success", "Fail", "Indeterminate"
rating (str, optional): The rating for the session.
"""
- valid_results = set(vars(SessionState).values())
- if end_state not in valid_results:
- raise ValueError(
- f"end_state must be one of {valid_results}. Provided: {end_state}")
+ SessionState(end_state=end_state)
self.end_state = end_state
self.rating = rating
self.end_timestamp = get_ISO_time()
diff --git a/requirements.txt b/requirements.txt
index dd5a435b..45f6e9f6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
pytest==7.4.0
requests==2.31.0
-requests-mock==1.11.0
\ No newline at end of file
+requests-mock==1.11.0
+pydantic==2.4.2
\ No newline at end of file
diff --git a/tests/mock_agent.py b/tests/mock_agent.py
index d8713d15..b25a7186 100644
--- a/tests/mock_agent.py
+++ b/tests/mock_agent.py
@@ -3,14 +3,27 @@ import time
print('init')
-ao_client = agentops.Client(api_key='floof', tags=['mock tests'])
+ao_client = agentops.Client(api_key='floof',
+ tags=['mock tests'])
+
+
+@ao_client.record_action('action')
+def sleep_func(sleep):
+ time.sleep(sleep)
+ print('1')
+
+
+@ao_client.record_action('multi')
+def multi_event(sleep):
+ sleep_func(1)
+ time.sleep(sleep)
print('Action 1')
-ao_client.record_action('Action 1')
-time.sleep(0.1)
+# ao_client.record_action('Action 1')
+sleep_func(0.1)
print('Action 2 - 5 second sleep')
-ao_client.record_action('Action 2')
-time.sleep(5)
+# ao_client.record_action('Action 2')
+multi_event(3)
print('End Session')
ao_client.end_session(end_state='Fail')
diff --git a/tests/test_canary.py b/tests/test_canary.py
index d3d74fcb..60b0ebde 100644
--- a/tests/test_canary.py
+++ b/tests/test_canary.py
@@ -2,7 +2,7 @@ import pytest
import requests_mock
import time
-from agentops import Client, Event, SessionState
+from agentops import Client, Event
@pytest.fixture
@@ -21,7 +21,7 @@ class TestCanary:
self.client = Client(api_key=self.api_key, max_wait_time=5)
def teardown_method(self):
- self.client.end_session(end_state=SessionState.SUCCESS)
+ self.client.end_session(end_state='Success')
def test_agent_ops_record(self, mock_req):
# Arrange
diff --git a/tests/test_logger.py b/tests/test_logger.py
index a8fe94a5..6ad66faf 100755
--- a/tests/test_logger.py
+++ b/tests/test_logger.py
@@ -1,7 +1,7 @@
import pytest
import requests_mock
import time
-from agentops import Client, AgentOpsLogger, SessionState
+from agentops import Client, AgentOpsLogger
@pytest.fixture
@@ -20,7 +20,7 @@ class TestLogger:
self.client = Client(api_key=self.api_key, max_wait_time=5)
def teardown_method(self):
- self.client.end_session(end_state=SessionState.SUCCESS)
+ self.client.end_session(end_state='Success')
def test_info(self, mock_req):
# Arrange
diff --git a/tests/test_session.py b/tests/test_session.py
index eeab3696..8ff26553 100644
--- a/tests/test_session.py
+++ b/tests/test_session.py
@@ -2,7 +2,7 @@ import pytest
import requests_mock
import time
-from agentops import Client, Event, EventState
+from agentops import Client, Event
@pytest.fixture
@@ -40,7 +40,7 @@ class TestSessions:
assert request_json['events'][0]['event_type'] == self.event_type
# Act
- end_state = EventState.SUCCESS
+ end_state = 'Success'
client.end_session(end_state)
time.sleep(0.1)
@@ -68,7 +68,7 @@ class TestSessions:
assert request_json['events'][0]['event_type'] == self.event_type
# Act
- end_state = EventState.SUCCESS
+ end_state = 'Success'
client.end_session(end_state)
time.sleep(0.1)
@@ -89,7 +89,7 @@ class TestRecordAction:
self.client = Client(self.api_key, max_wait_time=5)
def teardown_method(self):
- self.client.end_session(end_state=EventState.SUCCESS)
+ self.client.end_session(end_state='Success')
def test_record_action_decorator(self, mock_req):
@self.client.record_action(event_name=self.event_type, tags=['foo', 'bar'])
@@ -107,7 +107,7 @@ class TestRecordAction:
assert request_json['events'][0]['event_type'] == self.event_type
assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}
assert request_json['events'][0]['returns'] == 7
- assert request_json['events'][0]['result'] == EventState.SUCCESS
+ assert request_json['events'][0]['result'] == 'Success'
assert request_json['events'][0]['tags'] == ['foo', 'bar']
def test_record_action_decorator_multiple(self, mock_req):
@@ -127,7 +127,7 @@ class TestRecordAction:
assert request_json['events'][0]['event_type'] == self.event_type
assert request_json['events'][0]['params'] == {'x': 1, 'y': 2, 'z': 3}
assert request_json['events'][0]['returns'] == 6
- assert request_json['events'][0]['result'] == EventState.SUCCESS
+ assert request_json['events'][0]['result'] == 'Success'
assert request_json['events'][0]['tags'] == ['foo', 'bar']
def test_llm_call(self, mock_req):
@@ -148,7 +148,7 @@ class TestRecordAction:
assert request_json['events'][0]['action_type'] == 'llm'
assert request_json['events'][0]['prompt'] == prompt
assert request_json['events'][0]['returns'] == 'output'
- assert request_json['events'][0]['result'] == EventState.SUCCESS
+ assert request_json['events'][0]['result'] == 'Success'
def test_llm_call_no_prompt(self, mock_req):
# Arrange
@@ -190,4 +190,4 @@ class TestRecordAction:
assert request_json['events'][0]['action_type'] == 'llm'
assert request_json['events'][0]['prompt'] == prompt
assert request_json['events'][0]['returns'] == 'output'
- assert request_json['events'][0]['result'] == EventState.SUCCESS
+ assert request_json['events'][0]['result'] == 'Success'