Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
41 async events get logged as instances (#42)
Alex Reibman039be9e11315
diff --git a/agentops/client.py b/agentops/client.pyindex 48057b65..cec565b6 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -116,55 +116,108 @@ class Client:event_name (str): The name of the event to record.tags (List[str], optional): Any tags associated with the event. Defaults to None."""+def decorator(func):- @functools.wraps(func)- def wrapper(*args, **kwargs):- init_time = get_ISO_time()- func_args = inspect.signature(func).parameters- arg_names = list(func_args.keys())- # Get default values- arg_values = {name: func_args[name].default- for name in arg_names if func_args[name].default- is not inspect._empty}- # Update with positional arguments- arg_values.update(dict(zip(arg_names, args)))- arg_values.update(kwargs)-- try:- returns = func(*args, **kwargs)-- # If the function returns multiple values, record them all in the same event- if isinstance(returns, tuple):- returns = list(returns)-- # Record the event after the function call- self.record(Event(event_type=event_name,- params=arg_values,- returns=returns,- result='Success',- action_type='action',- init_timestamp=init_time,- tags=tags))-- except Exception as e:- # Record the event after the function call- self.record(Event(event_type=event_name,- params=arg_values,- returns=None,- result='Fail',- action_type='action',- init_timestamp=init_time,- tags=tags))-- # Re-raise the exception- raise-- return returns-- return wrapper+ if inspect.iscoroutinefunction(func):+ @functools.wraps(func)+ async def async_wrapper(*args, **kwargs):+ return await self._record_event_async(func, event_name, tags, *args, **kwargs)+ else:+ @functools.wraps(func)+ def sync_wrapper(*args, **kwargs):+ return self._record_event_sync(func, event_name, tags, *args, **kwargs)++ return async_wrapper if inspect.iscoroutinefunction(func) else sync_wrapperreturn decorator+ def _record_event_sync(self, func, event_name, tags, *args, **kwargs):+ init_time = get_ISO_time()+ func_args = inspect.signature(func).parameters+ arg_names = list(func_args.keys())+ # Get default values+ arg_values = {name: func_args[name].default+ for name in arg_names if func_args[name].default+ is not inspect._empty}+ # Update with positional arguments+ arg_values.update(dict(zip(arg_names, args)))+ arg_values.update(kwargs)++ try:+ returns = func(*args, **kwargs)++ # If the function returns multiple values, record them all in the same event+ if isinstance(returns, tuple):+ returns = list(returns)++ # Record the event after the function call+ self.record(Event(event_type=event_name,+ params=arg_values,+ returns=returns,+ result='Success',+ action_type='action',+ init_timestamp=init_time,+ tags=tags))++ except Exception as e:+ # Record the event after the function call+ self.record(Event(event_type=event_name,+ params=arg_values,+ returns=None,+ result='Fail',+ action_type='action',+ init_timestamp=init_time,+ tags=tags))++ # Re-raise the exception+ raise++ return returns++ async def _record_event_async(self, func, event_name, tags, *args, **kwargs):+ init_time = get_ISO_time()+ func_args = inspect.signature(func).parameters+ arg_names = list(func_args.keys())+ # Get default values+ arg_values = {name: func_args[name].default+ for name in arg_names if func_args[name].default+ is not inspect._empty}+ # Update with positional arguments+ arg_values.update(dict(zip(arg_names, args)))+ arg_values.update(kwargs)++ try:++ returns = await func(*args, **kwargs)++ # If the function returns multiple values, record them all in the same event+ if isinstance(returns, tuple):+ returns = list(returns)++ # Record the event after the function call+ self.record(Event(event_type=event_name,+ params=arg_values,+ returns=returns,+ result='Success',+ action_type='action',+ init_timestamp=init_time,+ tags=tags))++ except Exception as e:+ # Record the event after the function call+ self.record(Event(event_type=event_name,+ params=arg_values,+ returns=None,+ result='Fail',+ action_type='action',+ init_timestamp=init_time,+ tags=tags))++ # Re-raise the exception+ raise++ return returns+def start_session(self, tags: Optional[List[str]] = None):"""Start a new session for recording events.diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex b52ff313..56abbdd6 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -46,6 +46,8 @@ class LlmTracker:if finish_reason:self.event_stream.returns['finish_reason'] = finish_reason+ # Update end_timestamp+ self.event_stream.end_timestamp = get_ISO_time()self.client.record(self.event_stream)self.event_stream = Noneexcept:diff --git a/pyproject.toml b/pyproject.tomlindex 85b457f9..9d6efdad 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"[project]name = "agentops"-version = "0.0.5"+version = "0.0.6"authors = [{ name="Alex Reibman", email="[email protected]" },{ name="Shawn Qiu", email="[email protected]" }diff --git a/tests/async_call_agent.py b/tests/async_call_agent.pyindex 7e83781f..95f95e6c 100644--- a/tests/async_call_agent.py+++ b/tests/async_call_agent.py@@ -1,12 +1,13 @@++import openaiimport agentopsimport asyncio-import openai+import osao_client = agentops.Client(api_key='',tags=['mock tests'])--openai.api_key = '3'+openai.api_key = os.environ['OPENAI_API_KEY']async def stream_achat():diff --git a/tests/mock_agent.py b/tests/mock_agent.pyindex b25a7186..50a046a8 100644--- a/tests/mock_agent.py+++ b/tests/mock_agent.py@@ -1,16 +1,36 @@import agentopsimport time+import asyncio+import os+from dotenv import load_dotenv++load_dotenv()+print('init')-ao_client = agentops.Client(api_key='floof',+ao_client = agentops.Client(api_key=os.environ['AGENTOPS_API_KEY'],tags=['mock tests'])@ao_client.record_action('action')def sleep_func(sleep):time.sleep(sleep)- print('1')+ print('sync sleep')+ try:+ raise ValueError+ except:+ ...+++@ao_client.record_action('async')+async def sleep_func_async(sleep):+ await asyncio.sleep(sleep)+ print('async sleep')+ try:+ raise ValueError+ except:+ ...@ao_client.record_action('multi')@@ -19,11 +39,32 @@ def multi_event(sleep):time.sleep(sleep)-print('Action 1')-# ao_client.record_action('Action 1')-sleep_func(0.1)-print('Action 2 - 5 second sleep')-# ao_client.record_action('Action 2')-multi_event(3)-print('End Session')-ao_client.end_session(end_state='Fail')+async def main():++ print('Action 1')+ try:+ sleep_func(0.1)+ except:+ pass++ try:+ multi_event(3)+ except:+ pass++ task1 = asyncio.create_task(sleep_func_async(3))+ task2 = asyncio.create_task(sleep_func_async(3))+ await task1+ await task2++ print('Action 1')+ try:+ sleep_func(0.1)+ except:+ pass+ print('End Session')+ ao_client.end_session(end_state='Success')+++if __name__ == '__main__':+ asyncio.run(main())diff --git a/tests/test_session.py b/tests/test_session.pyindex 8ff26553..e4d7da05 100644--- a/tests/test_session.py+++ b/tests/test_session.py@@ -1,6 +1,8 @@import pytestimport requests_mockimport time+from datetime import datetime+from agentops import Client, Event@@ -130,64 +132,50 @@ class TestRecordAction:assert request_json['events'][0]['result'] == 'Success'assert request_json['events'][0]['tags'] == ['foo', 'bar']- def test_llm_call(self, mock_req):- # Arrange- prompt = 'prompt'+ @pytest.mark.asyncio+ async def test_async_function_call(self, mock_req):- @self.client.record_action(event_name=self.event_type, action_type='llm', model='gpt-4')- def llm_call(prompt=prompt):- return 'output'+ @self.client.record_action(self.event_type)+ async def async_add(x, y):+ time.sleep(0.1)+ return x + y# Act- llm_call()+ result = await async_add(3, 4)time.sleep(0.1)+ # Assert+ assert result == 7# Assertassert len(mock_req.request_history) == 1+ assert mock_req.last_request.headers['X-Agentops-Auth'] == self.api_keyrequest_json = mock_req.last_request.json()- assert request_json['events'][0]['action_type'] == 'llm'- assert request_json['events'][0]['prompt'] == prompt- assert request_json['events'][0]['returns'] == 'output'+ assert request_json['events'][0]['event_type'] == self.event_type+ assert request_json['events'][0]['params'] == {'x': 3, 'y': 4}+ assert request_json['events'][0]['returns'] == 7assert request_json['events'][0]['result'] == 'Success'+ init = datetime.fromisoformat(+ request_json['events'][0]['init_timestamp'].replace('Z', '+00:00'))+ end = datetime.fromisoformat(+ request_json['events'][0]['end_timestamp'].replace('Z', '+00:00'))- def test_llm_call_no_prompt(self, mock_req):- # Arrange- @self.client.record_action(event_name=self.event_type,- action_type='llm', model='gpt-4')- def llm_call():- return 'output'-- # Act and Assert- with pytest.raises(ValueError):- llm_call()-- def test_llm_call_no_model(self, mock_req):- # Arrange- prompt = 'prompt'-- @self.client.record_action(event_name=self.event_type, action_type='llm')- def llm_call(prompt=prompt):- return 'output'+ assert (end - init).total_seconds() >= 0.1- # Act and Assert- with pytest.raises(ValueError):- llm_call()-- def test_llm_call_no_action_type(self, mock_req):+ def test_function_call(self, mock_req):# Arrangeprompt = 'prompt'- @self.client.record_action(event_name=self.event_type, model='gpt-4')- def llm_call(prompt=prompt):+ @self.client.record_action(event_name=self.event_type)+ def foo(prompt=prompt):return 'output'- llm_call()+ # Act+ foo()time.sleep(0.1)# Assertassert len(mock_req.request_history) == 1request_json = mock_req.last_request.json()- assert request_json['events'][0]['action_type'] == 'llm'- assert request_json['events'][0]['prompt'] == prompt+ assert request_json['events'][0]['action_type'] == 'action'assert request_json['events'][0]['returns'] == 'output'assert request_json['events'][0]['result'] == 'Success'