Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Working state for chatml support

Howard Gil43a26e1a57be
diff --git a/agentops/__init__.py b/agentops/__init__.py
index d5ab55d1..3544b612 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -3,5 +3,5 @@
from .client import Client
from .event import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
from .logger import AgentOpsLogger
-from .enums import Models
+from .enums import Models, PromptMessageFormat
from .decorators import record_function
diff --git a/agentops/enums.py b/agentops/enums.py
index 35cf596e..ac0d087b 100644
--- a/agentops/enums.py
+++ b/agentops/enums.py
@@ -27,3 +27,8 @@ class Models(Enum):
GPT_4_32K_0314 = "gpt-4-32k-0314"
GPT_4_0613 = "gpt-4-0613"
TEXT_EMBEDDING_ADA_002 = "text-embedding-ada-002"
+
+
+class PromptMessageFormat(Enum):
+ STRING = "string"
+ CHATML = "chatml"
diff --git a/agentops/event.py b/agentops/event.py
index 6930de79..34c395c0 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -8,8 +8,9 @@ Data Class:
from dataclasses import dataclass, field
from typing import List, Optional
from .helpers import get_ISO_time
-from .enums import EventType, Models
+from .enums import EventType, Models, PromptMessageFormat
from uuid import UUID, uuid4
+import logging
@dataclass
@@ -42,12 +43,29 @@ class LLMEvent(Event):
event_type: str = EventType.LLM.value
agent_id: Optional[UUID] = None
thread_id: Optional[UUID] = None
- prompt: Optional[str] = None
+ prompt_messages: str | object = None
+ prompt_messages_format: PromptMessageFormat = PromptMessageFormat.STRING
+ # TODO: remove and just create it in __post_init__?
+ _formatted_prompt_messages: object = field(init=False, default=None)
completion: Optional[str] = None
model: Optional[Models] = None
prompt_tokens: Optional[int] = None
completion_tokens: Optional[int] = None
+ def __post_init__(self):
+ if self.prompt_messages_format == PromptMessageFormat.STRING:
+ self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}
+ elif self.prompt_messages_format == PromptMessageFormat.CHATML:
+ # Check if prompt_messages is already a list (indicating direct messages without "messages" key)
+ if isinstance(self.prompt_messages, list):
+ # Direct list of messages, add under "messages" key
+ self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}
+ elif "messages" in self.prompt_messages:
+ # prompt_messages is a dict that includes a "messages" key
+ self._formatted_prompt_messages = {"type": "chatml", **self.prompt_messages}
+ else:
+ logging.error("AgentOps: invalid prompt_messages format")
+
@dataclass
class ToolEvent(Event):
@@ -60,18 +78,18 @@ class ToolEvent(Event):
# Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details
@dataclass
class ErrorEvent():
- trigger_event_id: Optional[UUID] = None
- trigger_event_type: Optional[EventType] = None
+ trigger_event: Optional[Event] = None
error_type: Optional[str] = None
code: Optional[str] = None
details: Optional[str] = None
logs: Optional[str] = None
+ # event_type: str = EventType.ERROR.value # TODO: don't expose this
+ event_type: str = field(init=False, default=EventType.ERROR.value)
+ timestamp: str = field(default_factory=get_ISO_time)
- def __init__(self, event: Event = None, **kwargs):
- self.event_type: str = EventType.ERROR.value
- self.timestamp = get_ISO_time()
- if event:
- self.trigger_event_id = event.id
- self.trigger_event_type = event.event_type
- for key, value in kwargs.items():
- setattr(self, key, value)
+ def __post_init__(self):
+ if self.trigger_event:
+ self.trigger_event_id = self.trigger_event.id
+ self.trigger_event_type = self.trigger_event.event_type
+ # TODO: remove trigger_event from serialization
+ # e.g. field(repr=False, compare=False, hash=False, metadata={'serialize': False})
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
index 49eedb9d..2e160261 100644
--- a/agentops/llm_tracker.py
+++ b/agentops/llm_tracker.py
@@ -6,6 +6,7 @@ import logging
from .event import LLMEvent
from .helpers import get_ISO_time, check_call_stack_for_agent_id
import inspect
+from .enums import PromptMessageFormat
class LlmTracker:
@@ -29,7 +30,7 @@ class LlmTracker:
def _handle_response_v0_openai(self, response, kwargs, init_timestamp):
"""Handle responses for OpenAI versions <v1.0.0"""
- prompt = kwargs.pop("messages", None) # pull prompt out for visibility
+ prompt_messages = {"messages": kwargs.pop("messages", [])} # pull messages out for visibility
def handle_stream_chunk(chunk):
try:
@@ -45,7 +46,8 @@ class LlmTracker:
returns={"finish_reason": None,
"content": token},
agent_id=check_call_stack_for_agent_id(),
- prompt=prompt,
+ prompt_messages=prompt_messages,
+ prompt_messages_format=PromptMessageFormat.CHATML,
completion=token,
model=model
)
@@ -64,7 +66,7 @@ class LlmTracker:
except Exception as e:
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
if inspect.isasyncgen(response):
@@ -91,7 +93,8 @@ class LlmTracker:
returns={"content":
response['choices'][0]['message']['content']},
agent_id=check_call_stack_for_agent_id(),
- prompt=prompt,
+ prompt_messages=prompt_messages,
+ prompt_messages_format=PromptMessageFormat.CHATML,
completion=response['choices'][0]['message']['content'],
model=response['model'],
prompt_tokens=response.get('usage',
@@ -111,7 +114,8 @@ class LlmTracker:
"content": response.choices[0].message.model_dump()},
agent_id=check_call_stack_for_agent_id(),
# TODO: Will need to make the completion the key for content, splat out the model dump
- prompt=prompt,
+ prompt_messages=prompt_messages,
+ prompt_messages_format=PromptMessageFormat.CHATML,
completion=response.choices[0].message.model_dump(),
model=response.model,
prompt_tokens=response.usage.prompt_tokens,
@@ -121,7 +125,7 @@ class LlmTracker:
except Exception as e:
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")
return response
@@ -131,7 +135,7 @@ class LlmTracker:
from openai.types.chat import ChatCompletionChunk
from openai.resources import AsyncCompletions
- prompt = kwargs.pop("messages", None) # pull prompt out for visibility
+ prompt_messages = {"messages": kwargs.pop("messages", [])} # pull messages out for visibility
def handle_stream_chunk(chunk: ChatCompletionChunk):
try:
@@ -148,7 +152,8 @@ class LlmTracker:
init_timestamp=init_timestamp,
params=kwargs,
agent_id=check_call_stack_for_agent_id(),
- prompt=prompt,
+ prompt_messages=prompt_messages,
+ prompt_messages_format=PromptMessageFormat.CHATML,
returns={"finish_reason": None,
"content": token},
model=model
@@ -172,7 +177,7 @@ class LlmTracker:
except Exception as e:
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
if isinstance(response, Stream):
@@ -207,7 +212,8 @@ class LlmTracker:
# TODO: Will need to make the completion the key for content, splat out the model dump
"content": response.choices[0].message.model_dump()},
agent_id=check_call_stack_for_agent_id(),
- prompt=prompt,
+ prompt_messages=prompt_messages,
+ prompt_messages_format=PromptMessageFormat.CHATML,
completion=response.choices[0].message.model_dump(),
model=response.model,
prompt_tokens=response.usage.prompt_tokens,
@@ -217,7 +223,7 @@ class LlmTracker:
except Exception as e:
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")
return response
diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_async.py b/tests/openai_handlers/test_llm_tracker_ge_1_async.py
index a216445c..de78fa4c 100644
--- a/tests/openai_handlers/test_llm_tracker_ge_1_async.py
+++ b/tests/openai_handlers/test_llm_tracker_ge_1_async.py
@@ -28,7 +28,7 @@ async def call_openai_async():
)
print(response)
- raise ValueError("This is an intentional error for testing.")
+ # raise ValueError("This is an intentional error for testing.")
async def main():
diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py
index 7fcfba19..b8e01879 100644
--- a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py
+++ b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py
@@ -39,7 +39,7 @@ def call_openai():
)
print(response)
- raise ValueError("This is an intentional error for testing.")
+ # raise ValueError("This is an intentional error for testing.")
call_openai()
diff --git a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/test_llm_tracker_lt_1_sync.py
index 70f917bf..9f33104c 100644
--- a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py
+++ b/tests/openai_handlers/test_llm_tracker_lt_1_sync.py
@@ -27,7 +27,7 @@ def call_openai():
model='gpt-3.5-turbo', messages=message, temperature=0.5)
print(response)
- raise ValueError("This is an intentional error for testing.")
+ # raise ValueError("This is an intentional error for testing.")
call_openai()