Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

WIP. Adding support for completions but needs more thought

Howard Gil79dad7316046
diff --git a/agentops/__init__.py b/agentops/__init__.py
index 3544b612..40ffa69d 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -3,5 +3,5 @@
from .client import Client
from .event import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
from .logger import AgentOpsLogger
-from .enums import Models, PromptMessageFormat
+from .enums import Models, LLMMessageFormat
from .decorators import record_function
diff --git a/agentops/client.py b/agentops/client.py
index 762f1505..7701397b 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -142,7 +142,7 @@ class Client(metaclass=MetaClient):
except Exception as e:
# TODO: add the stack trace
- self.record(ErrorEvent(event=event, details={f"{type(e).__name__}": str(e)}))
+ self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
# Re-raise the exception
raise
@@ -182,7 +182,7 @@ class Client(metaclass=MetaClient):
except Exception as e:
# TODO: add the stack trace
- self.record(ErrorEvent(event=event, details={f"{type(e).__name__}": str(e)}))
+ self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))
# Re-raise the exception
raise
diff --git a/agentops/enums.py b/agentops/enums.py
index ac0d087b..8f704968 100644
--- a/agentops/enums.py
+++ b/agentops/enums.py
@@ -29,6 +29,6 @@ class Models(Enum):
TEXT_EMBEDDING_ADA_002 = "text-embedding-ada-002"
-class PromptMessageFormat(Enum):
+class LLMMessageFormat(Enum):
STRING = "string"
CHATML = "chatml"
diff --git a/agentops/event.py b/agentops/event.py
index 34c395c0..f8df6836 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -8,7 +8,7 @@ Data Class:
from dataclasses import dataclass, field
from typing import List, Optional
from .helpers import get_ISO_time
-from .enums import EventType, Models, PromptMessageFormat
+from .enums import EventType, Models, LLMMessageFormat
from uuid import UUID, uuid4
import logging
@@ -21,6 +21,7 @@ class Event:
init_timestamp: Optional[str] = field(default_factory=get_ISO_time)
end_timestamp: str = field(default_factory=get_ISO_time)
id: UUID = field(default_factory=uuid4)
+ # TODO: has_been_recorded: bool = False
@dataclass
@@ -43,28 +44,51 @@ class LLMEvent(Event):
event_type: str = EventType.LLM.value
agent_id: Optional[UUID] = None
thread_id: Optional[UUID] = None
- prompt_messages: str | object = None
- prompt_messages_format: PromptMessageFormat = PromptMessageFormat.STRING
- # TODO: remove and just create it in __post_init__?
+ prompt_messages: str | object = None # TODO: remove from serialization
+ prompt_messages_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization
+ # TODO: remove and just create it in __post_init__ so it can never be set by user?
_formatted_prompt_messages: object = field(init=False, default=None)
- completion: Optional[str] = None
+ completion_message: str | object = None # TODO: remove from serialization
+ completion_message_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization
+ # TODO: remove and just create it in __post_init__ so it can never be set by user?
+ _formatted_completion_message: object = field(init=False, default=None)
model: Optional[Models] = None
prompt_tokens: Optional[int] = None
completion_tokens: Optional[int] = None
def __post_init__(self):
- if self.prompt_messages_format == PromptMessageFormat.STRING:
+ # TODO can we just figure out if it's chatml so user doesn't have to pass anything?
+ if self.prompt_messages_format == LLMMessageFormat.STRING:
self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}
- elif self.prompt_messages_format == PromptMessageFormat.CHATML:
+ elif self.prompt_messages_format == LLMMessageFormat.CHATML:
# Check if prompt_messages is already a list (indicating direct messages without "messages" key)
if isinstance(self.prompt_messages, list):
# Direct list of messages, add under "messages" key
+ # [{'role': 'system', 'content': '...'}]
self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}
elif "messages" in self.prompt_messages:
# prompt_messages is a dict that includes a "messages" key
+ # {'messages': [{'role': 'system', 'content': '...'}]}
self._formatted_prompt_messages = {"type": "chatml", **self.prompt_messages}
else:
- logging.error("AgentOps: invalid prompt_messages format")
+ logging.error("AgentOps: invalid prompt_messages")
+
+ if self.completion_message_format == LLMMessageFormat.STRING:
+ self._formatted_completion_message = {"type": "string", "string": self.completion_message}
+ elif self.completion_message_format == LLMMessageFormat.CHATML:
+ # Check if completion_message is already a list (indicating direct messages without "messages" key)
+ if "message" in self.completion_message:
+ # completion_message is a dict that includes a "messages" key
+ # {'message': {'role': 'assistant', 'content': '...'}}
+ self._formatted_completion_message = {"type": "chatml", **self.completion_message}
+ elif True: # TODO
+ # Direct list of messages, add under "messages" key
+ # [{'role': 'system', 'content': '...'}]
+ self._formatted_completion_message = {"type": "chatml", "messages": self.completion_message}
+ else:
+ logging.error("AgentOps: invalid completion_message")
+
+ print("\n\ncompletion:\n", self.completion, "\n\n")
@dataclass
@@ -78,7 +102,7 @@ class ToolEvent(Event):
# Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details
@dataclass
class ErrorEvent():
- trigger_event: Optional[Event] = None
+ trigger_event: Optional[Event] = None # TODO: remove from serialization?
error_type: Optional[str] = None
code: Optional[str] = None
details: Optional[str] = None
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
index 2e160261..0eabb85b 100644
--- a/agentops/llm_tracker.py
+++ b/agentops/llm_tracker.py
@@ -6,7 +6,7 @@ import logging
from .event import LLMEvent
from .helpers import get_ISO_time, check_call_stack_for_agent_id
import inspect
-from .enums import PromptMessageFormat
+from .enums import LLMMessageFormat
class LlmTracker:
@@ -30,7 +30,7 @@ class LlmTracker:
def _handle_response_v0_openai(self, response, kwargs, init_timestamp):
"""Handle responses for OpenAI versions <v1.0.0"""
- prompt_messages = {"messages": kwargs.pop("messages", [])} # pull messages out for visibility
+ prompt_messages = kwargs.pop("messages") # pull messages out for visibility
def handle_stream_chunk(chunk):
try:
@@ -47,7 +47,7 @@ class LlmTracker:
"content": token},
agent_id=check_call_stack_for_agent_id(),
prompt_messages=prompt_messages,
- prompt_messages_format=PromptMessageFormat.CHATML,
+ prompt_messages_format=LLMMessageFormat.CHATML,
completion=token,
model=model
)
@@ -94,7 +94,7 @@ class LlmTracker:
response['choices'][0]['message']['content']},
agent_id=check_call_stack_for_agent_id(),
prompt_messages=prompt_messages,
- prompt_messages_format=PromptMessageFormat.CHATML,
+ prompt_messages_format=LLMMessageFormat.CHATML,
completion=response['choices'][0]['message']['content'],
model=response['model'],
prompt_tokens=response.get('usage',
@@ -115,7 +115,7 @@ class LlmTracker:
agent_id=check_call_stack_for_agent_id(),
# TODO: Will need to make the completion the key for content, splat out the model dump
prompt_messages=prompt_messages,
- prompt_messages_format=PromptMessageFormat.CHATML,
+ prompt_messages_format=LLMMessageFormat.CHATML,
completion=response.choices[0].message.model_dump(),
model=response.model,
prompt_tokens=response.usage.prompt_tokens,
@@ -135,7 +135,7 @@ class LlmTracker:
from openai.types.chat import ChatCompletionChunk
from openai.resources import AsyncCompletions
- prompt_messages = {"messages": kwargs.pop("messages", [])} # pull messages out for visibility
+ prompt_messages = kwargs.pop("messages") # pull messages out for visibility
def handle_stream_chunk(chunk: ChatCompletionChunk):
try:
@@ -153,7 +153,7 @@ class LlmTracker:
params=kwargs,
agent_id=check_call_stack_for_agent_id(),
prompt_messages=prompt_messages,
- prompt_messages_format=PromptMessageFormat.CHATML,
+ prompt_messages_format=LLMMessageFormat.CHATML,
returns={"finish_reason": None,
"content": token},
model=model
@@ -205,16 +205,24 @@ class LlmTracker:
# v1.0.0+ responses are objects
try:
+
+ import json
+ test = response.choices[0].message
+ json_string = json.dumps(test)
+ print(json_string)
+ test = response.choices[0].message.model_dump()
+ json_string = json.dumps(test)
+ print(json_string)
+
self.client.record(LLMEvent(
init_timestamp=init_timestamp,
params=kwargs,
- returns={
- # TODO: Will need to make the completion the key for content, splat out the model dump
- "content": response.choices[0].message.model_dump()},
+ returns=response,
agent_id=check_call_stack_for_agent_id(),
prompt_messages=prompt_messages,
- prompt_messages_format=PromptMessageFormat.CHATML,
- completion=response.choices[0].message.model_dump(),
+ prompt_messages_format=LLMMessageFormat.CHATML,
+ completion_message=response.choices[0].message.model_dump(),
+ completion_message_format=LLMMessageFormat.CHATML,
model=response.model,
prompt_tokens=response.usage.prompt_tokens,
completion_tokens=response.usage.completion_tokens