Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
removed tags. Tidied up event
Howard Gil256c541c9ba5
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex f9ea5569..f560821b 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -1,6 +1,6 @@# agentops/__init__.pyfrom .client import Client-from .event import ActionEvent, LLMEvent, ToolEvent, Error+from .event import ActionEvent, LLMEvent, ToolEvent, ErrorEventfrom .logger import AgentOpsLoggerfrom .enums import Modelsdiff --git a/agentops/client.py b/agentops/client.pyindex 0857ed36..6933629c 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -5,7 +5,7 @@ Classes:Client: Provides methods to interact with the AgentOps service."""-from .event import Event, ActionEvent, Error+from .event import Event, ActionEvent, ErrorEventfrom .helpers import get_ISO_time, singleton, check_call_stack_for_agent_idfrom .session import Sessionfrom .worker import Worker@@ -110,7 +110,7 @@ class Client(metaclass=MetaClient):logging.warning("AgentOps: Cannot record event - no current session")- def record_function(self, event_name: str, tags: Optional[List[str]] = None):+ def record_function(self, event_name: str):"""Decorator to record an event before and after a function call.Usage:@@ -119,26 +119,25 @@ class Client(metaclass=MetaClient):the action is recordedArgs:event_name (str): The name of the event to record.- tags (List[str], optional): Any tags associated with the event. Defaults to None."""def decorator(func):if inspect.iscoroutinefunction(func):@functools.wraps(func)async def async_wrapper(*args, **kwargs):- return await self._record_event_async(func, event_name, tags, *args, **kwargs)+ return await self._record_event_async(func, event_name, *args, **kwargs)return async_wrapperelse:@functools.wraps(func)def sync_wrapper(*args, **kwargs):- return self._record_event_sync(func, event_name, tags, *args, **kwargs)+ return self._record_event_sync(func, event_name, *args, **kwargs)return sync_wrapperreturn decorator- def _record_event_sync(self, func, event_name, tags, *args, **kwargs):+ def _record_event_sync(self, func, event_name, *args, **kwargs):init_time = get_ISO_time()func_args = inspect.signature(func).parametersarg_names = list(func_args.keys())@@ -150,8 +149,7 @@ class Client(metaclass=MetaClient):arg_values.update(dict(zip(arg_names, args)))arg_values.update(kwargs)- event = ActionEvent(tags=tags,- params=arg_values,+ event = ActionEvent(params=arg_values,init_timestamp=init_time,agent_id=check_call_stack_for_agent_id(),action_type=event_name)@@ -164,20 +162,22 @@ class Client(metaclass=MetaClient):returns = list(returns)event.returns = returns+ event.end_timestamp = get_ISO_time()# TODO: If func excepts this will never get called# the dev loses all the useful stuff in ActionEvent they would need for debugging# we should either record earlier or have Error post the supplied event to supabaseself.record(event)except Exception as e:- self.record(Error(event=event, details={f"{type(e).__name__}": str(e)}))+ # TODO: add the stack trace+ self.record(ErrorEvent(event=event, details={f"{type(e).__name__}": str(e)}))# Re-raise the exceptionraisereturn returns- async def _record_event_async(self, func, event_name, tags, *args, **kwargs):+ async def _record_event_async(self, func, event_name, *args, **kwargs):init_time = get_ISO_time()func_args = inspect.signature(func).parametersarg_names = list(func_args.keys())@@ -189,8 +189,7 @@ class Client(metaclass=MetaClient):arg_values.update(dict(zip(arg_names, args)))arg_values.update(kwargs)- event = ActionEvent(tags=tags,- params=arg_values,+ event = ActionEvent(params=arg_values,init_timestamp=init_time,agent_id=check_call_stack_for_agent_id(),action_type=event_name)@@ -202,14 +201,16 @@ class Client(metaclass=MetaClient):if isinstance(returns, tuple):returns = list(returns)+ event.returns = returns+ event.end_timestamp = get_ISO_time()# TODO: If func excepts this will never get called# the dev loses all the useful stuff in ActionEvent they would need for debugging# we should either record earlier or have Error post the supplied event to supabase- event.returns = returnsself.record(event)except Exception as e:- self.record(Error(event=event, details={f"{type(e).__name__}": str(e)}))+ # TODO: add the stack trace+ self.record(ErrorEvent(event=event, details={f"{type(e).__name__}": str(e)}))# Re-raise the exceptionraisediff --git a/agentops/enums.py b/agentops/enums.pyindex 138de2d0..35cf596e 100644--- a/agentops/enums.py+++ b/agentops/enums.py@@ -6,6 +6,7 @@ class EventType(Enum):ACTION = "actions"API = "apis"TOOL = "tools"+ ERROR = "errors"class Result(Enum):diff --git a/agentops/event.py b/agentops/event.pyindex bbfb2989..6930de79 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -15,7 +15,6 @@ from uuid import UUID, uuid4@dataclassclass Event:event_type: str # EventType.ENUM.value- tags: Optional[List[str]] = Noneparams: Optional[str] = Nonereturns: Optional[str] = Noneinit_timestamp: Optional[str] = field(default_factory=get_ISO_time)@@ -58,9 +57,9 @@ class ToolEvent(Event):logs: Optional[str] = None-####################+# Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details@dataclass-class Error():+class ErrorEvent():trigger_event_id: Optional[UUID] = Nonetrigger_event_type: Optional[EventType] = Noneerror_type: Optional[str] = None@@ -69,7 +68,7 @@ class Error():logs: Optional[str] = Nonedef __init__(self, event: Event = None, **kwargs):- self.event_type = "errors" # Temporary to accomodate /events endpoint. Won't be necessary with /errors endpoint+ self.event_type: str = EventType.ERROR.valueself.timestamp = get_ISO_time()if event:self.trigger_event_id = event.iddiff --git a/tests/openai_handlers/test_llm_tracker_ge_1_async.py b/tests/openai_handlers/test_llm_tracker_ge_1_async.pynew file mode 100644index 00000000..e4f445c6--- /dev/null+++ b/tests/openai_handlers/test_llm_tracker_ge_1_async.py@@ -0,0 +1,40 @@+from openai import AsyncOpenAI # Assuming AsyncOpenAI is the async version of the client+import asyncio+import agentops+from packaging.version import parse+from importlib import import_module+import sys++ao_client = agentops.Client()+++@ao_client.record_function('sample function being recorded')+async def call_openai_async():+ client = AsyncOpenAI() # Using the async client++ response = await client.chat.completions.create( # Await the async call+ model="gpt-3.5-turbo",+ messages=[+ {+ "role": "system",+ "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."+ },+ {+ "role": "user",+ "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"+ }+ ],+ temperature=0.7,+ max_tokens=64,+ top_p=1+ )++ print(response)+ raise ValueError("This is an intentional error for testing.")+++async def main():+ await call_openai_async()+ ao_client.end_session('Success') # This would also need to be made async if it makes network calls++asyncio.run(main())diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/test_llm_tracker_ge_1_sync.pynew file mode 100644index 00000000..f4a17ce4--- /dev/null+++ b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py@@ -0,0 +1,46 @@+from openai import OpenAI+import agentops+from packaging.version import parse+from importlib import import_module+import sys+++ao_client = agentops.Client()++api = 'openai'+if api in sys.modules:+ module = import_module(api)+ if api == 'openai':+ if hasattr(module, '__version__'):+ module_version = parse(module.__version__)+ print('openai version: ', module_version)+++@ao_client.record_function('sample function being recorded')+def call_openai():+ client = OpenAI()++ response = client.chat.completions.create(+ model="gpt-3.5-turbo",+ messages=[+ {+ "role": "system",+ "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."+ },+ {+ "role": "user",+ "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"+ }+ ],+ temperature=0.7,+ max_tokens=64,+ top_p=1+ )++ print(response)+ raise ValueError("This is an intentional error for testing.")+++call_openai()++ao_client.end_session('Success')diff --git a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/test_llm_tracker_lt_1_sync.pynew file mode 100644index 00000000..10297ad4--- /dev/null+++ b/tests/openai_handlers/test_llm_tracker_lt_1_sync.py@@ -0,0 +1,35 @@+import openai+from openai import ChatCompletion+import agentops+from packaging.version import parse+from importlib import import_module+import sys+++ao_client = agentops.Client()++api = 'openai'+if api in sys.modules:+ module = import_module(api)+ if api == 'openai':+ if hasattr(module, '__version__'):+ module_version = parse(module.__version__)+ print('openai version: ', module_version)+++@ao_client.record_function('sample function being recorded')+def call_openai():++ message = [{"role": "user", "content": "Hello"},+ {"role": "assistant", "content": "Hi there!"}]++ response = ChatCompletion.create(+ model='gpt-3.5-turbo', messages=message, temperature=0.5)++ print(response)+ raise ValueError("This is an intentional error for testing.")+++call_openai()++ao_client.end_session('Success')diff --git a/tests/test_session.py b/tests/test_session.pyindex 8b958d43..8a0e37d4 100644--- a/tests/test_session.py+++ b/tests/test_session.py@@ -94,7 +94,7 @@ class TestRecordAction:self.client.end_session(end_state='Success')def test_record_function_decorator(self, mock_req):- @self.client.record_function(event_name=self.event_type, tags=['foo', 'bar'])+ @self.client.record_function(event_name=self.event_type)def add_two(x, y):return x + y@@ -114,7 +114,7 @@ class TestRecordAction:def test_record_function_decorator_multiple(self, mock_req):# Arrange- @self.client.record_function(event_name=self.event_type, tags=['foo', 'bar'])+ @self.client.record_function(event_name=self.event_type)def add_three(x, y, z=3):return x + y + z