Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

removed tags. Tidied up event

Howard Gil256c541c9ba5
diff --git a/agentops/__init__.py b/agentops/__init__.py
index f9ea5569..f560821b 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -1,6 +1,6 @@
# agentops/__init__.py
from .client import Client
-from .event import ActionEvent, LLMEvent, ToolEvent, Error
+from .event import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
from .logger import AgentOpsLogger
from .enums import Models
diff --git a/agentops/client.py b/agentops/client.py
index 0857ed36..6933629c 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -5,7 +5,7 @@ Classes:
Client: Provides methods to interact with the AgentOps service.
"""
-from .event import Event, ActionEvent, Error
+from .event import Event, ActionEvent, ErrorEvent
from .helpers import get_ISO_time, singleton, check_call_stack_for_agent_id
from .session import Session
from .worker import Worker
@@ -110,7 +110,7 @@ class Client(metaclass=MetaClient):
logging.warning(
"AgentOps: Cannot record event - no current session")
- def record_function(self, event_name: str, tags: Optional[List[str]] = None):
+ def record_function(self, event_name: str):
"""
Decorator to record an event before and after a function call.
Usage:
@@ -119,26 +119,25 @@ class Client(metaclass=MetaClient):
the action is recorded
Args:
event_name (str): The name of the event to record.
- tags (List[str], optional): Any tags associated with the event. Defaults to None.
"""
def decorator(func):
if inspect.iscoroutinefunction(func):
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
- return await self._record_event_async(func, event_name, tags, *args, **kwargs)
+ return await self._record_event_async(func, event_name, *args, **kwargs)
return async_wrapper
else:
@functools.wraps(func)
def sync_wrapper(*args, **kwargs):
- return self._record_event_sync(func, event_name, tags, *args, **kwargs)
+ return self._record_event_sync(func, event_name, *args, **kwargs)
return sync_wrapper
return decorator
- def _record_event_sync(self, func, event_name, tags, *args, **kwargs):
+ def _record_event_sync(self, func, event_name, *args, **kwargs):
init_time = get_ISO_time()
func_args = inspect.signature(func).parameters
arg_names = list(func_args.keys())
@@ -150,8 +149,7 @@ class Client(metaclass=MetaClient):
arg_values.update(dict(zip(arg_names, args)))
arg_values.update(kwargs)
- event = ActionEvent(tags=tags,
- params=arg_values,
+ event = ActionEvent(params=arg_values,
init_timestamp=init_time,
agent_id=check_call_stack_for_agent_id(),
action_type=event_name)
@@ -164,20 +162,22 @@ class Client(metaclass=MetaClient):
returns = list(returns)
event.returns = returns
+ event.end_timestamp = get_ISO_time()
# TODO: If func excepts this will never get called
# the dev loses all the useful stuff in ActionEvent they would need for debugging
# we should either record earlier or have Error post the supplied event to supabase
self.record(event)
except Exception as e:
- self.record(Error(event=event, details={f"{type(e).__name__}": str(e)}))
+ # TODO: add the stack trace
+ self.record(ErrorEvent(event=event, details={f"{type(e).__name__}": str(e)}))
# Re-raise the exception
raise
return returns
- async def _record_event_async(self, func, event_name, tags, *args, **kwargs):
+ async def _record_event_async(self, func, event_name, *args, **kwargs):
init_time = get_ISO_time()
func_args = inspect.signature(func).parameters
arg_names = list(func_args.keys())
@@ -189,8 +189,7 @@ class Client(metaclass=MetaClient):
arg_values.update(dict(zip(arg_names, args)))
arg_values.update(kwargs)
- event = ActionEvent(tags=tags,
- params=arg_values,
+ event = ActionEvent(params=arg_values,
init_timestamp=init_time,
agent_id=check_call_stack_for_agent_id(),
action_type=event_name)
@@ -202,14 +201,16 @@ class Client(metaclass=MetaClient):
if isinstance(returns, tuple):
returns = list(returns)
+ event.returns = returns
+ event.end_timestamp = get_ISO_time()
# TODO: If func excepts this will never get called
# the dev loses all the useful stuff in ActionEvent they would need for debugging
# we should either record earlier or have Error post the supplied event to supabase
- event.returns = returns
self.record(event)
except Exception as e:
- self.record(Error(event=event, details={f"{type(e).__name__}": str(e)}))
+ # TODO: add the stack trace
+ self.record(ErrorEvent(event=event, details={f"{type(e).__name__}": str(e)}))
# Re-raise the exception
raise
diff --git a/agentops/enums.py b/agentops/enums.py
index 138de2d0..35cf596e 100644
--- a/agentops/enums.py
+++ b/agentops/enums.py
@@ -6,6 +6,7 @@ class EventType(Enum):
ACTION = "actions"
API = "apis"
TOOL = "tools"
+ ERROR = "errors"
class Result(Enum):
diff --git a/agentops/event.py b/agentops/event.py
index bbfb2989..6930de79 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -15,7 +15,6 @@ from uuid import UUID, uuid4
@dataclass
class Event:
event_type: str # EventType.ENUM.value
- tags: Optional[List[str]] = None
params: Optional[str] = None
returns: Optional[str] = None
init_timestamp: Optional[str] = field(default_factory=get_ISO_time)
@@ -58,9 +57,9 @@ class ToolEvent(Event):
logs: Optional[str] = None
-####################
+# Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details
@dataclass
-class Error():
+class ErrorEvent():
trigger_event_id: Optional[UUID] = None
trigger_event_type: Optional[EventType] = None
error_type: Optional[str] = None
@@ -69,7 +68,7 @@ class Error():
logs: Optional[str] = None
def __init__(self, event: Event = None, **kwargs):
- self.event_type = "errors" # Temporary to accomodate /events endpoint. Won't be necessary with /errors endpoint
+ self.event_type: str = EventType.ERROR.value
self.timestamp = get_ISO_time()
if event:
self.trigger_event_id = event.id
diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_async.py b/tests/openai_handlers/test_llm_tracker_ge_1_async.py
new file mode 100644
index 00000000..e4f445c6
--- /dev/null
+++ b/tests/openai_handlers/test_llm_tracker_ge_1_async.py
@@ -0,0 +1,40 @@
+from openai import AsyncOpenAI # Assuming AsyncOpenAI is the async version of the client
+import asyncio
+import agentops
+from packaging.version import parse
+from importlib import import_module
+import sys
+
+ao_client = agentops.Client()
+
+
+@ao_client.record_function('sample function being recorded')
+async def call_openai_async():
+ client = AsyncOpenAI() # Using the async client
+
+ response = await client.chat.completions.create( # Await the async call
+ model="gpt-3.5-turbo",
+ messages=[
+ {
+ "role": "system",
+ "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."
+ },
+ {
+ "role": "user",
+ "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"
+ }
+ ],
+ temperature=0.7,
+ max_tokens=64,
+ top_p=1
+ )
+
+ print(response)
+ raise ValueError("This is an intentional error for testing.")
+
+
+async def main():
+ await call_openai_async()
+ ao_client.end_session('Success') # This would also need to be made async if it makes network calls
+
+asyncio.run(main())
diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py
new file mode 100644
index 00000000..f4a17ce4
--- /dev/null
+++ b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py
@@ -0,0 +1,46 @@
+from openai import OpenAI
+import agentops
+from packaging.version import parse
+from importlib import import_module
+import sys
+
+
+ao_client = agentops.Client()
+
+api = 'openai'
+if api in sys.modules:
+ module = import_module(api)
+ if api == 'openai':
+ if hasattr(module, '__version__'):
+ module_version = parse(module.__version__)
+ print('openai version: ', module_version)
+
+
+@ao_client.record_function('sample function being recorded')
+def call_openai():
+ client = OpenAI()
+
+ response = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[
+ {
+ "role": "system",
+ "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."
+ },
+ {
+ "role": "user",
+ "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"
+ }
+ ],
+ temperature=0.7,
+ max_tokens=64,
+ top_p=1
+ )
+
+ print(response)
+ raise ValueError("This is an intentional error for testing.")
+
+
+call_openai()
+
+ao_client.end_session('Success')
diff --git a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/test_llm_tracker_lt_1_sync.py
new file mode 100644
index 00000000..10297ad4
--- /dev/null
+++ b/tests/openai_handlers/test_llm_tracker_lt_1_sync.py
@@ -0,0 +1,35 @@
+import openai
+from openai import ChatCompletion
+import agentops
+from packaging.version import parse
+from importlib import import_module
+import sys
+
+
+ao_client = agentops.Client()
+
+api = 'openai'
+if api in sys.modules:
+ module = import_module(api)
+ if api == 'openai':
+ if hasattr(module, '__version__'):
+ module_version = parse(module.__version__)
+ print('openai version: ', module_version)
+
+
+@ao_client.record_function('sample function being recorded')
+def call_openai():
+
+ message = [{"role": "user", "content": "Hello"},
+ {"role": "assistant", "content": "Hi there!"}]
+
+ response = ChatCompletion.create(
+ model='gpt-3.5-turbo', messages=message, temperature=0.5)
+
+ print(response)
+ raise ValueError("This is an intentional error for testing.")
+
+
+call_openai()
+
+ao_client.end_session('Success')
diff --git a/tests/test_session.py b/tests/test_session.py
index 8b958d43..8a0e37d4 100644
--- a/tests/test_session.py
+++ b/tests/test_session.py
@@ -94,7 +94,7 @@ class TestRecordAction:
self.client.end_session(end_state='Success')
def test_record_function_decorator(self, mock_req):
- @self.client.record_function(event_name=self.event_type, tags=['foo', 'bar'])
+ @self.client.record_function(event_name=self.event_type)
def add_two(x, y):
return x + y
@@ -114,7 +114,7 @@ class TestRecordAction:
def test_record_function_decorator_multiple(self, mock_req):
# Arrange
- @self.client.record_function(event_name=self.event_type, tags=['foo', 'bar'])
+ @self.client.record_function(event_name=self.event_type)
def add_three(x, y, z=3):
return x + y + z