Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Eng-113-events-docstring (#128)

Howard Giladf66d7ea871
diff --git a/agentops/__init__.py b/agentops/__init__.py
index 09efb23f..e09375a0 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -4,9 +4,8 @@ from typing import Optional, List
from .client import Client
from .config import Configuration
-from .event import ActionEvent, LLMEvent, ToolEvent, ErrorEvent, Event
from .event import Event, ActionEvent, LLMEvent, ToolEvent, ErrorEvent
-from .enums import Models, LLMMessageFormat, EndState
+from .enums import Models
from .decorators import record_function
from os import environ
diff --git a/agentops/client.py b/agentops/client.py
index 7580b8ca..9b802f5f 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -47,6 +47,7 @@ class Client(metaclass=MetaClient):
auto_start_session (bool): Whether to start a session automatically when the client is created.
Attributes:
_session (Session, optional): A Session is a grouping of events (e.g. a run of your agent).
+ _worker (Worker, optional): A Worker manages the event queue and sends session updates to the AgentOps api server
"""
def __init__(self,
diff --git a/agentops/enums.py b/agentops/enums.py
index 1d6d6677..2c93be15 100644
--- a/agentops/enums.py
+++ b/agentops/enums.py
@@ -23,11 +23,6 @@ class Models(Enum):
TEXT_EMBEDDING_ADA_002 = "text-embedding-ada-002"
-class LLMMessageFormat(Enum):
- STRING = "string"
- CHATML = "chatml"
-
-
class EndState(Enum):
SUCCESS = "Success"
FAIL = "Fail"
diff --git a/agentops/event.py b/agentops/event.py
index 02102b49..b4b10d0e 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -8,12 +8,33 @@ Data Class:
from dataclasses import dataclass, field
from typing import List, Optional
from .helpers import get_ISO_time
-from .enums import EventType, Models, LLMMessageFormat
+from .enums import EventType, Models
from uuid import UUID, uuid4
@dataclass
class Event:
+
+ """
+ Abstract base class for events that will be recorded. Should not be instantiated directly.
+
+ event_type(str): The type of event. Defined in enums.EventType. Some values are 'llm', 'action', 'api', 'tool', 'error'.
+ params(dict, optional): The parameters of the function containing the triggered event, e.g. {'x': 1} in example below
+ returns(str, optional): The return value of the function containing the triggered event, e.g. 2 in example below
+ init_timestamp(str): A timestamp indicating when the event began. Defaults to the time when this Event was instantiated.
+ end_timestamp(str): A timestamp indicating when the event ended. Defaults to the time when this Event was instantiated.
+ id(UUID): A unique identifier for the event. Defaults to a new UUID.
+
+ foo(x=1) {
+ ...
+ // params equals {'x': 1}
+ record(ActionEvent(params=**kwargs, ...))
+ ...
+ // returns equals 2
+ return x+1
+ }
+ """
+
event_type: str # EventType.ENUM.value
params: Optional[dict] = None
returns: Optional[str] = None
@@ -25,6 +46,15 @@ class Event:
@dataclass
class ActionEvent(Event):
+ """
+ For generic events
+
+ agent_id(UUID, optional): The unique identifier of the agent that triggered the event.
+ action_type(str, optional): High level name describing the action
+ logs(str, optional): For detailed information/logging related to the action
+ screenshot(str, optional): url to snapshot if agent interacts with UI
+ """
+
event_type: str = EventType.ACTION.value
# TODO: Should not be optional, but non-default argument 'agent_id' follows default argument error
agent_id: Optional[UUID] = None
@@ -40,51 +70,63 @@ class ActionEvent(Event):
@dataclass
class LLMEvent(Event):
+
+ """
+ For recording calls to LLMs. AgentOps auto-instruments calls to the most popular LLMs e.g. GPT, Claude, Gemini, etc.
+
+ agent_id(UUID, optional): The unique identifier of the agent that triggered the event.
+ thread_id(UUID, optional): The unique identifier of the contextual thread that a message pertains to.
+ prompt_message(str, list, optional): The message or messages that were used to prompt the LLM. Preferably in ChatML format which is more fully supported by AgentOps.
+ prompt_tokens(int, optional): The number of tokens in the prompt message.
+ completion_message(str, object, optional): The message or returned by the LLM. Preferably in ChatML format which is more fully supported by AgentOps.
+ completion_tokens(int, optional): The number of tokens in the completion message.
+ model(Models, str, optional): LLM model e.g. "gpt-4". Models defined in enums.Models are more fully supported by AgentOps e.g. extra features in dashboard.
+
+ """
+
event_type: str = EventType.LLM.value
agent_id: Optional[UUID] = None
thread_id: Optional[UUID] = None
- prompt_messages: str | List = None # TODO: remove from serialization
- prompt_messages_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization
- # TODO: remove and just create it in __post_init__ so it can never be set by user?
- _formatted_prompt_messages: object = field(init=False, default=None)
- completion_message: str | object = None # TODO: remove from serialization
- completion_message_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization
- # TODO: remove and just create it in __post_init__ so it can never be set by user?
- _formatted_completion_message: object = field(init=False, default=None)
- model: Optional[Models | str] = None
+ prompt_message: str | List = None
prompt_tokens: Optional[int] = None
+ completion_message: str | object = None
completion_tokens: Optional[int] = None
-
- def format_messages(self):
- if self.prompt_messages:
- # TODO should we just figure out if it's chatml so user doesn't have to pass anything?
- if self.prompt_messages_format == LLMMessageFormat.STRING:
- self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}
- elif self.prompt_messages_format == LLMMessageFormat.CHATML:
- self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}
-
- if self.completion_message:
- if self.completion_message_format == LLMMessageFormat.STRING:
- self._formatted_completion_message = {"type": "string", "string": self.completion_message}
- elif self.completion_message_format == LLMMessageFormat.CHATML:
- self._formatted_completion_message = {"type": "chatml", "message": self.completion_message}
-
- def __post_init__(self):
- # format if prompt/completion messages were passed when LLMEvent was created
- self.format_messages()
+ model: Optional[Models | str] = None
@dataclass
class ToolEvent(Event):
+ """
+ For recording calls to tools e.g. searchWeb, fetchFromDB
+
+ agent_id(UUID, optional): The unique identifier of the agent that triggered the event.
+ name(str, optional): A name describing the tool or the actual function name if applicable e.g. searchWeb, fetchFromDB.
+ logs(str, dict, optional): For detailed information/logging related to the tool.
+
+ """
event_type: str = EventType.TOOL.value
agent_id: Optional[UUID] = None
name: Optional[str] = None
logs: Optional[str | dict] = None
-
# Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details
+
+
@dataclass
class ErrorEvent():
+
+ """
+ For recording any errors e.g. ones related to agent execution
+
+ trigger_event(Event, optional): The event object that triggered the error if applicable.
+ error_type(str, optional): The type of error e.g. "ValueError".
+ code(str, optional): A code that can be used to identify the error e.g. 501.
+ details(str, optional): Detailed information about the error.
+ logs(str, optional): For detailed information/logging related to the error.
+ timestamp(str): A timestamp indicating when the error occurred. Defaults to the time when this ErrorEvent was instantiated.
+
+ """
+
trigger_event: Optional[Event] = None # TODO: remove from serialization?
error_type: Optional[str] = None
code: Optional[str] = None
diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.py
index 4a05f50d..daca246d 100644
--- a/agentops/langchain_callback_handler.py
+++ b/agentops/langchain_callback_handler.py
@@ -12,7 +12,6 @@ from tenacity import RetryCallState
from agentops import Client as AOClient
from agentops import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
-from agentops import LLMMessageFormat
from agentops.helpers import get_ISO_time
from .helpers import debug_print_function_params
@@ -65,7 +64,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):
**({} if metadata is None else metadata),
**kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logs
model=kwargs['invocation_params']['model'],
- prompt_messages=prompts[0]
+ prompt_message=prompts[0]
# tags=tags # TODO
)
@@ -101,16 +100,14 @@ class LangchainCallbackHandler(BaseCallbackHandler):
llm_event.end_timestamp = get_ISO_time()
if response.llm_output is not None:
llm_event.completion_message = response.generations[0][0].message.content # TODO
- llm_event.completion_message_format = LLMMessageFormat.STRING # TODO
llm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']
- llm_event.format_messages() # TODO: Find somewhere logical to call this on the user's behalf. They shouldn't call it
self.ao_client.record(llm_event)
if len(response.generations) == 0:
# TODO: more descriptive error
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],
- details="on_llm_end: No generations", timestamp=get_ISO_time())
+ details="on_llm_end: No generations", timestamp=get_ISO_time())
self.ao_client.record(error_event)
@debug_print_function_params
@@ -366,7 +363,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
**({} if metadata is None else metadata),
**kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logs
model=kwargs['invocation_params']['model'],
- prompt_messages=prompts[0]
+ prompt_message=prompts[0]
)
@debug_print_function_params
@@ -428,16 +425,14 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
llm_event.end_timestamp = get_ISO_time()
if response.llm_output is not None:
llm_event.completion_message = response.generations[0][0].message.content # TODO
- llm_event.completion_message_format = LLMMessageFormat.STRING # TODO
llm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']
- llm_event.format_messages() # TODO: Find somewhere logical to call this on the user's behalf. They shouldn't call it
self.ao_client.record(llm_event)
if len(response.generations) == 0:
# TODO: more descriptive error
error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],
- details="on_llm_end: No generations", timestamp=get_ISO_time())
+ details="on_llm_end: No generations", timestamp=get_ISO_time())
self.ao_client.record(error_event)
@debug_print_function_params
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
index ed9ce6cf..6b03c840 100644
--- a/agentops/llm_tracker.py
+++ b/agentops/llm_tracker.py
@@ -6,7 +6,6 @@ import logging
from .event import LLMEvent, ErrorEvent
from .helpers import get_ISO_time, check_call_stack_for_agent_id
import inspect
-from .enums import LLMMessageFormat
class LlmTracker:
@@ -41,6 +40,7 @@ class LlmTracker:
)
try:
+ # NOTE: prompt/completion usage not returned in response when streaming
model = chunk['model']
choices = chunk['choices']
token = choices[0]['delta'].get('content', '')
@@ -50,14 +50,11 @@ class LlmTracker:
if finish_reason:
self.llm_event.agent_id = check_call_stack_for_agent_id()
- self.llm_event.prompt_messages = kwargs["messages"]
- self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML
+ self.llm_event.prompt_message = kwargs["messages"]
self.llm_event.completion_message = {"role": "assistant", "content": self.completion}
- self.llm_event.completion_message_format = LLMMessageFormat.CHATML
self.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion}
self.llm_event.model = model
self.llm_event.end_timestamp = get_ISO_time()
- self.llm_event.format_messages()
self.client.record(self.llm_event)
except Exception as e:
@@ -90,14 +87,13 @@ class LlmTracker:
# v0.0.0 responses are dicts
try:
self.llm_event.agent_id = check_call_stack_for_agent_id()
- self.llm_event.prompt_messages = kwargs["messages"]
- self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML
- self.llm_event.completion_message = response['choices'][0]['message']
- self.llm_event.completion_message_format = LLMMessageFormat.CHATML
+ self.llm_event.prompt_message = kwargs["messages"]
+ self.llm_event.prompt_tokens = response['usage']['prompt_tokens']
+ self.llm_event.completion_message = response['choices'][0]['message']['content']
+ self.llm_event.completion_tokens = response['usage']['completion_tokens']
self.llm_event.returns = {"content": response['choices'][0]['message']['content']}
self.llm_event.model = response["model"]
self.llm_event.end_timestamp = get_ISO_time()
- self.llm_event.format_messages()
self.client.record(self.llm_event)
except Exception as e:
@@ -125,6 +121,7 @@ class LlmTracker:
)
try:
+ # NOTE: prompt/completion usage not returned in response when streaming
model = chunk.model
choices = chunk.choices
token = choices[0].delta.content
@@ -137,15 +134,12 @@ class LlmTracker:
if finish_reason:
self.llm_event.agent_id = check_call_stack_for_agent_id()
- self.llm_event.prompt_messages = kwargs["messages"]
- self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML
+ self.llm_event.prompt_message = kwargs["messages"]
self.llm_event.completion_message = {"role": "assistant", "content": self.completion}
- self.llm_event.completion_message_format = LLMMessageFormat.CHATML
self.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion,
"function_call": function_call, "tool_calls": tool_calls, "role": role}
self.llm_event.model = model
self.llm_event.end_timestamp = get_ISO_time()
- self.llm_event.format_messages()
self.client.record(self.llm_event)
except Exception as e:
@@ -185,13 +179,12 @@ class LlmTracker:
# v1.0.0+ responses are objects
try:
self.llm_event.agent_id = check_call_stack_for_agent_id()
- self.llm_event.prompt_messages = kwargs["messages"]
- self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML
- self.llm_event.completion_message = response.choices[0].message.model_dump()
- self.llm_event.completion_message_format = LLMMessageFormat.CHATML
+ self.llm_event.prompt_message = kwargs["messages"]
+ self.llm_event.prompt_tokens = response.usage.prompt_tokens
+ self.llm_event.completion_message = response.choices[0].message.model_dump().get('content')
+ self.llm_event.completion_tokens = response.usage.completion_tokens
self.llm_event.returns = response.model_dump()
self.llm_event.model = response.model
- self.llm_event.format_messages()
self.client.record(self.llm_event)
except Exception as e:
diff --git a/tests/openai_handlers/_test_llm_tracker_ge_1_async.py b/tests/openai_handlers/_test_llm_tracker_ge_1_async.py
index de78fa4c..8d580094 100644
--- a/tests/openai_handlers/_test_llm_tracker_ge_1_async.py
+++ b/tests/openai_handlers/_test_llm_tracker_ge_1_async.py
@@ -1,38 +1,64 @@
-from openai import AsyncOpenAI # Assuming AsyncOpenAI is the async version of the client
+from openai import AsyncOpenAI
import asyncio
import agentops
from agentops import record_function
+from dotenv import load_dotenv
-ao_client = agentops.Client()
+load_dotenv()
+agentops.init()
-@record_function('sample function being recorded')
-async def call_openai_async():
+
+@record_function('openai v1 async no streaming')
+async def call_openai_v1_async_no_streaming():
+ client = AsyncOpenAI()
+
+ chat_completion = await client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[
+ {
+ "role": "system",
+ "content": "You are an expert haiku writer"
+ },
+ {
+ "role": "user",
+ "content": "write me a haiku about devops"
+ }
+ ]
+ )
+
+ print(chat_completion)
+ # raise ValueError("This is an intentional error for testing.")
+
+
+@record_function('openai v1 async with streaming')
+async def call_openai_v1_async_streaming():
client = AsyncOpenAI() # Using the async client
- response = await client.chat.completions.create( # Await the async call
+ chat_completion = await client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{
"role": "system",
- "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."
+ "content": "You are an expert haiku writer"
},
{
"role": "user",
- "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"
+ "content": "write me a haiku about devops"
}
],
- temperature=0.7,
- max_tokens=64,
- top_p=1
+ stream=True
)
- print(response)
+ async for chunk in chat_completion:
+ chunk_message = chunk.choices[0].delta.content
+ print(chunk_message)
# raise ValueError("This is an intentional error for testing.")
async def main():
- await call_openai_async()
- ao_client.end_session('Success') # This would also need to be made async if it makes network calls
+ await call_openai_v1_async_no_streaming()
+ await call_openai_v1_async_streaming()
+ agentops.end_session('Success') # This would also need to be made async if it makes network calls
asyncio.run(main())
diff --git a/tests/openai_handlers/_test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/_test_llm_tracker_ge_1_sync.py
index 415e0af0..e602e5c6 100644
--- a/tests/openai_handlers/_test_llm_tracker_ge_1_sync.py
+++ b/tests/openai_handlers/_test_llm_tracker_ge_1_sync.py
@@ -4,9 +4,11 @@ from agentops import record_function
from packaging.version import parse
from importlib import import_module
import sys
+from dotenv import load_dotenv
+load_dotenv()
-ao_client = agentops.Client()
+agentops.init()
api = 'openai'
if api in sys.modules:
@@ -17,11 +19,31 @@ if api in sys.modules:
print('openai version: ', module_version)
-@record_function('sample function being recorded')
-def call_openai():
+@record_function('openai v1 sync no streaming')
+def call_openai_v1_sync_no_streaming():
client = OpenAI()
+ chat_completion = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[
+ {
+ "role": "system",
+ "content": "You are an expert haiku writer"
+ },
+ {
+ "role": "user",
+ "content": "write me a haiku about devops"
+ }
+ ]
+ )
+
+ print(chat_completion)
+ # raise ValueError("This is an intentional error for testing.")
+
- response = client.chat.completions.create(
+@record_function('openai v1 sync with streaming')
+def call_openai_v1_sync_streaming():
+ client = OpenAI()
+ chat_completion = client.chat.completions.create(
model="gpt-3.5-turbo",
messages=[
{
@@ -33,15 +55,17 @@ def call_openai():
"content": "write me a haiku about devops"
}
],
- temperature=0.7,
- max_tokens=64,
- top_p=1
+ stream=True
)
- print(response)
+ for chunk in chat_completion:
+ chunk_message = chunk.choices[0].delta.content
+ print(chunk_message)
+
# raise ValueError("This is an intentional error for testing.")
-call_openai()
+call_openai_v1_sync_no_streaming()
+call_openai_v1_sync_streaming()
-ao_client.end_session('Success')
+agentops.end_session('Success')
diff --git a/tests/openai_handlers/_test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/_test_llm_tracker_lt_1_sync.py
index 9f33104c..5555ad76 100644
--- a/tests/openai_handlers/_test_llm_tracker_lt_1_sync.py
+++ b/tests/openai_handlers/_test_llm_tracker_lt_1_sync.py
@@ -1,12 +1,14 @@
-import openai
from openai import ChatCompletion
import agentops
from agentops import record_function
from packaging.version import parse
from importlib import import_module
import sys
+from dotenv import load_dotenv
-ao_client = agentops.Client()
+load_dotenv()
+
+agentops.init()
api = 'openai'
if api in sys.modules:
@@ -17,19 +19,51 @@ if api in sys.modules:
print('openai version: ', module_version)
-@record_function('sample function being recorded')
-def call_openai():
+@record_function('openai v0 sync no streaming')
+def call_openai_v0_sync_no_streaming():
+ chat_completion = ChatCompletion.create(
+ model='gpt-3.5-turbo',
+ messages=[
+ {
+ "role": "system",
+ "content": "You are an expert haiku writer"
+ },
+ {
+ "role": "user",
+ "content": "write me a haiku about devops"
+ }
+ ]
+ )
+
+ print(chat_completion)
+ # raise ValueError("This is an intentional error for testing.")
- message = [{"role": "user", "content": "Hello"},
- {"role": "assistant", "content": "Hi there!"}]
- response = ChatCompletion.create(
- model='gpt-3.5-turbo', messages=message, temperature=0.5)
+@record_function('openai v0 sync with streaming')
+def call_openai_v0_sync_with_streaming():
+ chat_completion = ChatCompletion.create(
+ model='gpt-3.5-turbo',
+ messages=[
+ {
+ "role": "system",
+ "content": "You are an expert haiku writer"
+ },
+ {
+ "role": "user",
+ "content": "write me a haiku about devops"
+ }
+ ],
+ stream=True
+ )
- print(response)
+ for chunk in chat_completion:
+ print(chunk.choices[0])
+ chunk_message = chunk.choices[0].delta["content"]
+ print(chunk_message)
# raise ValueError("This is an intentional error for testing.")
-call_openai()
+call_openai_v0_sync_no_streaming()
+call_openai_v0_sync_with_streaming()
-ao_client.end_session('Success')
+agentops.end_session('Success')