Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
18 hanging thread after exit (#21)
Alex Reibman8d6c5771393e
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 037d98c5..d39a5a0f 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -1,4 +1,4 @@-from .agentops import AgentOps-from .event import Event, EventState, SessionState-from .config import Configuration+from .client import Client+from .event import Event, EventState+from .session import SessionStatefrom .logger import AgentOpsLoggerdiff --git a/agentops/agentops.py b/agentops/client.pysimilarity index 63%rename from agentops/agentops.pyrename to agentops/client.pyindex 301d85eb..c3bbe62c 100644--- a/agentops/agentops.py+++ b/agentops/client.py@@ -2,42 +2,68 @@AgentOps client module that provides a client class with public interfaces and configuration.Classes:- AgentOps: Provides methods to interact with the AgentOps service.+ Client: Provides methods to interact with the AgentOps service."""-from .config import Configuration-from .event import Session, Event, EventState+from .event import Event, EventState+from .session import Session, SessionStatefrom .worker import Workerfrom uuid import uuid4from typing import Optional, Listimport functoolsimport inspectimport atexit+import signal+import sys++from .config import Configuration-class AgentOps:+class Client:"""Client for AgentOps service.Args:api_key (str): API Key for AgentOps services.tags (List[str], optional): Tags for the sessions that can be used for grouping or sorting later (e.g. ["GPT-4"]).- config (Configuration, optional): A Configuration object for AgentOps services. If not provided, a default Configuration object will be used.-+ endpoint (str, optional): The endpoint for the AgentOps service. Defaults to 'https://agentops-server-v2.fly.dev'.+ max_wait_time (int, optional): The maximum time to wait in milliseconds before flushing the queue. Defaults to 1000.+ max_queue_size (int, optional): The maximum size of the event queue. Defaults to 100.Attributes:session (Session, optional): A Session is a grouping of events (e.g. a run of your agent).- config (Configuration): A Configuration object for AgentOps services."""- def __init__(self, api_key: str, tags: Optional[List[str]] = None, config: Configuration = Configuration()):+ def __init__(self, api_key: str, tags: Optional[List[str]] = None,+ endpoint: Optional[str] = 'https://agentops-server-v2.fly.dev',+ max_wait_time: Optional[int] = 1000,+ max_queue_size: Optional[int] = 100):++ # Create a worker config+ self.config = Configuration(api_key, endpoint,+ max_wait_time, max_queue_size)+# Store a reference to the instance- AgentOps._instance = self+ Client._instance = selfatexit.register(self.cleanup)- self.config: Configuration = config- self.config.api_key = api_key+ # Register signal handler for SIGINT (Ctrl+C) and SIGTERM+ signal.signal(signal.SIGINT, self.signal_handler)+ signal.signal(signal.SIGTERM, self.signal_handler)+self.start_session(tags)+ def signal_handler(self, signal, frame):+ """+ Signal handler for SIGINT (Ctrl+C) and SIGTERM. Ends the session and exits the program.++ Args:+ signal (int): The signal number.+ frame: The current stack frame.+ """+ print('Signal SIGTERM or SIGINT detected. Ending session...')+ self.end_session(end_state=EventState.FAIL)+ sys.exit(0)+def record(self, event: Event):"""Record an event with the AgentOps service.@@ -46,12 +72,11 @@ class AgentOps:event (Event): The event to record."""- if not self.session.has_ended():+ if not self.session.has_ended:self.worker.add_event(- {'session_id': self.session.get_session_id(), **event.__dict__})+ {'session_id': self.session.session_id, **event.__dict__})else:- print("This event was not recorded because the previous session has been ended. Start a new session to record again")- print(event.__dict__)+ print("This event was not recorded because the previous session has been ended. Start a new session to record again.")def record_action(self, event_name: str, tags: Optional[List[str]] = None):"""@@ -114,7 +139,7 @@ class AgentOps:self.worker = Worker(self.config)self.worker.start_session(self.session)- def end_session(self, end_state: EventState = EventState.INDETERMINATE, rating: Optional[str] = None):+ def end_session(self, end_state: SessionState = SessionState.INDETERMINATE, rating: Optional[str] = None):"""End the current session with the AgentOps service.@@ -122,15 +147,17 @@ class AgentOps:end_state (str, optional): The final state of the session.rating (str, optional): The rating for the session."""- valid_results = set(vars(EventState).values())+ valid_results = set(vars(SessionState).values())if end_state not in valid_results:raise ValueError(- f"end_state must be one of {EventState.__args__}. Provided: {end_state}")-- self.session.end_session(end_state, rating)- self.worker.end_session(self.session)+ f"end_state must be one of {SessionState}. Provided: {end_state}")+ if not self.session.has_ended:+ self.session.end_session(end_state, rating)+ self.worker.end_session(self.session)+ else:+ print("Warning: The session has already been ended.")def cleanup(self):# Only run cleanup function if session is created- if hasattr(self, "session"):- self.end_session(end_state=EventState.FAIL)+ if hasattr(self, "session") and not self.session.has_ended:+ self.end_session(end_state=SessionState.FAIL)diff --git a/agentops/event.py b/agentops/event.pyindex 612ae92c..de71acf8 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -3,7 +3,6 @@ AgentOps events.Classes:Event: Represents discrete events to be recorded.- Session: Represents a session of events, with a start and end state."""from .helpers import get_ISO_timefrom typing import Optional, List@@ -15,12 +14,6 @@ class EventState:INDETERMINATE = "Indeterminate"-class SessionState:- SUCCESS = "Success"- FAIL = "Fail"- INDETERMINATE = "Indeterminate"--class Event:"""Represents a discrete event to be recorded.@@ -49,59 +42,3 @@ class Event:self.result = resultself.tags = tagsself.timestamp = get_ISO_time()---class Session:- """- Represents a session of events, with a start and end state.-- Args:- session_id (str): The session id is used to record particular runs.- tags (List[str], optional): Tags that can be used for grouping or sorting later. Examples could be ["GPT-4"].-- Attributes:- init_timestamp (float): The timestamp for when the session started, represented as seconds since the epoch.- end_timestamp (float, optional): The timestamp for when the session ended, represented as seconds since the epoch. This is only set after end_session is called.- end_state (str, optional): The final state of the session. Suggested: "Success", "Fail", "Indeterminate"- rating (str, optional): The rating for the session.-- """-- def __init__(self, session_id: str, tags: Optional[List[str]] = None):- self.session_id = session_id- self.init_timestamp = get_ISO_time()- self.tags = tags-- def end_session(self, end_state: SessionState = SessionState.INDETERMINATE, rating: Optional[str] = None):- """- End the session with a specified state and rating.-- Args:- end_state (str, optional): The final state of the session. Suggested: "Success", "Fail", "Indeterminate"- rating (str, optional): The rating for the session.- """- valid_results = set(vars(SessionState).values())- if end_state not in valid_results:- raise ValueError(- f"end_state must be one of {valid_results}. Provided: {end_state}")- self.end_state = end_state- self.rating = rating- self.end_timestamp = get_ISO_time()-- def get_session_id(self) -> str:- """- Get the session id.-- Returns:- str: The id of the session.- """- return self.session_id-- def has_ended(self) -> str:- """- Returns whether the session has been ended-- Returns:- bool: Whether the session has been ended- """- return hasattr(self, "end_state")diff --git a/agentops/logger.py b/agentops/logger.pyindex 2f71cbb5..be2166fe 100755--- a/agentops/logger.py+++ b/agentops/logger.py@@ -1,6 +1,6 @@import loggingimport re-from .agentops import AgentOps+from .client import Clientfrom .event import Event, EventState@@ -14,8 +14,8 @@ class AgentOpsLogger():Example Usage:- >>> from agentops import AgentOps- >>> client = AgentOps(...)+ >>> from agentops import Client+ >>> client = Client(...)>>> logger = AgentOpsLogger.get_agentops_logger(client, 'my_logger')>>> logger.info('This is an info log')@@ -23,12 +23,12 @@ class AgentOpsLogger():"""@staticmethod- def get_agentops_logger(client: AgentOps, name: str, level=logging.DEBUG):+ def get_agentops_logger(client: Client, name: str, level=logging.DEBUG):"""Create and return a logger with an AgentOpsHandler.Args:- client (AgentOps): The AgentOps client to which the logs will be sent.+ client (Client): The AgentOps client to which the logs will be sent.name (str): The name for the logger and handler.level (int, optional): The minimum severity level to log. Defaults to logging.DEBUG.@@ -43,12 +43,12 @@ class AgentOpsLogger():return logger@staticmethod- def get_agentops_handler(client: AgentOps, name: str):+ def get_agentops_handler(client: Client, name: str):"""Create and return an AgentOpsHandler.Args:- client (AgentOps): The AgentOps client to which the logs will be sent.+ client (Client): The AgentOps client to which the logs will be sent.name (str): The name for the handler.Returns:@@ -65,12 +65,12 @@ class AgentOpsHandler(logging.Handler):It also removes ANSI color codes from log messages before sending them."""- def __init__(self, client: AgentOps, name: str):+ def __init__(self, client: Client, name: str):"""Initialize the handler with a specific AgentOps client and name.Args:- client (AgentOps): The AgentOps client to which the logs will be sent.+ client (Client): The AgentOps client to which the logs will be sent.name (str): The name for this handler."""super().__init__()diff --git a/agentops/session.py b/agentops/session.pynew file mode 100644index 00000000..ade8c945--- /dev/null+++ b/agentops/session.py@@ -0,0 +1,56 @@+from .helpers import get_ISO_time+from typing import Optional, List+++class SessionState:+ SUCCESS = "Success"+ FAIL = "Fail"+ INDETERMINATE = "Indeterminate"+++class Session:+ """+ Represents a session of events, with a start and end state.++ Args:+ session_id (str): The session id is used to record particular runs.+ tags (List[str], optional): Tags that can be used for grouping or sorting later. Examples could be ["GPT-4"].++ Attributes:+ init_timestamp (float): The timestamp for when the session started, represented as seconds since the epoch.+ end_timestamp (float, optional): The timestamp for when the session ended, represented as seconds since the epoch. This is only set after end_session is called.+ end_state (str, optional): The final state of the session. Suggested: "Success", "Fail", "Indeterminate"+ rating (str, optional): The rating for the session.++ """++ def __init__(self, session_id: str, tags: Optional[List[str]] = None):+ self.session_id = session_id+ self.init_timestamp = get_ISO_time()+ self.tags = tags++ def end_session(self, end_state: SessionState = SessionState.INDETERMINATE, rating: Optional[str] = None):+ """+ End the session with a specified state and rating.++ Args:+ end_state (str, optional): The final state of the session. Suggested: "Success", "Fail", "Indeterminate"+ rating (str, optional): The rating for the session.+ """+ valid_results = set(vars(SessionState).values())+ if end_state not in valid_results:+ raise ValueError(+ f"end_state must be one of {valid_results}. Provided: {end_state}")+ self.end_state = end_state+ self.rating = rating+ self.end_timestamp = get_ISO_time()++ @property+ def has_ended(self) -> bool:+ """+ Returns whether the session has been ended++ Returns:+ bool: Whether the session has been ended+ """+ return hasattr(self, "end_state")diff --git a/agentops/worker.py b/agentops/worker.pyindex c450efc3..75154821 100644--- a/agentops/worker.py+++ b/agentops/worker.py@@ -3,7 +3,7 @@ import threadingimport timefrom .http import HttpClientfrom .config import Configuration-from .event import Session+from .session import Sessionfrom typing import Dictdiff --git a/tests/mock_agent.py b/tests/mock_agent.pynew file mode 100644index 00000000..d8713d15--- /dev/null+++ b/tests/mock_agent.py@@ -0,0 +1,16 @@+import agentops+import time++print('init')++ao_client = agentops.Client(api_key='floof', tags=['mock tests'])+++print('Action 1')+ao_client.record_action('Action 1')+time.sleep(0.1)+print('Action 2 - 5 second sleep')+ao_client.record_action('Action 2')+time.sleep(5)+print('End Session')+ao_client.end_session(end_state='Fail')diff --git a/tests/test_canary.py b/tests/test_canary.pyindex a6fe44e0..d3d74fcb 100644--- a/tests/test_canary.py+++ b/tests/test_canary.py@@ -2,8 +2,7 @@ import pytestimport requests_mockimport time-from agentops import AgentOps, Event, Configuration-from agentops.event import EventState+from agentops import Client, Event, SessionState@pytest.fixture@@ -19,11 +18,10 @@ class TestCanary:def setup_method(self):self.url = 'https://agentops-server-v2.fly.dev'self.api_key = "random_api_key"- self.config = Configuration(max_wait_time=5)- self.client = AgentOps(api_key=self.api_key, config=self.config)+ self.client = Client(api_key=self.api_key, max_wait_time=5)def teardown_method(self):- self.client.end_session(end_state=EventState.SUCCESS)+ self.client.end_session(end_state=SessionState.SUCCESS)def test_agent_ops_record(self, mock_req):# Arrangediff --git a/tests/test_logger.py b/tests/test_logger.pyindex 7b1c0355..a8fe94a5 100755--- a/tests/test_logger.py+++ b/tests/test_logger.py@@ -1,8 +1,7 @@import pytestimport requests_mockimport time-from agentops import AgentOps, AgentOpsLogger, Configuration-from agentops.event import EventState+from agentops import Client, AgentOpsLogger, SessionState@pytest.fixture@@ -18,11 +17,10 @@ class TestLogger:def setup_method(self):self.url = 'https://agentops-server-v2.fly.dev'self.api_key = "random_api_key"- self.config = Configuration(max_wait_time=5)- self.client = AgentOps(api_key=self.api_key, config=self.config)+ self.client = Client(api_key=self.api_key, max_wait_time=5)def teardown_method(self):- self.client.end_session(end_state=EventState.SUCCESS)+ self.client.end_session(end_state=SessionState.SUCCESS)def test_info(self, mock_req):# Arrangediff --git a/tests/test_session.py b/tests/test_session.pyindex dd539f03..9be0d15c 100644--- a/tests/test_session.py+++ b/tests/test_session.py@@ -2,7 +2,7 @@ import pytestimport requests_mockimport time-from agentops import AgentOps, Event, Configuration, EventState+from agentops import Client, Event, EventState@pytest.fixture@@ -21,8 +21,7 @@ class TestSessions:def test_session(self, mock_req):# Arrange- config = Configuration(max_wait_time=5)- client = AgentOps(api_key=self.api_key, config=config)+ client = Client(api_key=self.api_key, max_wait_time=5)# Actclient.record(Event(self.event_type))@@ -56,8 +55,7 @@ class TestSessions:def test_tags(self, mock_req):# Arrangetags = ['GPT-4']- config = Configuration(max_wait_time=5)- client = AgentOps(api_key=self.api_key, config=config, tags=tags)+ client = Client(api_key=self.api_key, tags=tags, max_wait_time=5)# Actclient.record(Event(self.event_type))@@ -88,8 +86,7 @@ class TestRecordAction:self.url = 'https://agentops-server-v2.fly.dev'self.api_key = "random_api_key"self.event_type = 'test_event_type'- self.config = Configuration(max_wait_time=5)- self.client = AgentOps(self.api_key, config=self.config)+ self.client = Client(self.api_key, max_wait_time=5)def teardown_method(self):self.client.end_session(end_state=EventState.SUCCESS)