Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Working state for chatml support
Howard Gil43a26e1a57be
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex d5ab55d1..3544b612 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -3,5 +3,5 @@from .client import Clientfrom .event import ActionEvent, LLMEvent, ToolEvent, ErrorEventfrom .logger import AgentOpsLogger-from .enums import Models+from .enums import Models, PromptMessageFormatfrom .decorators import record_functiondiff --git a/agentops/enums.py b/agentops/enums.pyindex 35cf596e..ac0d087b 100644--- a/agentops/enums.py+++ b/agentops/enums.py@@ -27,3 +27,8 @@ class Models(Enum):GPT_4_32K_0314 = "gpt-4-32k-0314"GPT_4_0613 = "gpt-4-0613"TEXT_EMBEDDING_ADA_002 = "text-embedding-ada-002"+++class PromptMessageFormat(Enum):+ STRING = "string"+ CHATML = "chatml"diff --git a/agentops/event.py b/agentops/event.pyindex 6930de79..34c395c0 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -8,8 +8,9 @@ Data Class:from dataclasses import dataclass, fieldfrom typing import List, Optionalfrom .helpers import get_ISO_time-from .enums import EventType, Models+from .enums import EventType, Models, PromptMessageFormatfrom uuid import UUID, uuid4+import logging@dataclass@@ -42,12 +43,29 @@ class LLMEvent(Event):event_type: str = EventType.LLM.valueagent_id: Optional[UUID] = Nonethread_id: Optional[UUID] = None- prompt: Optional[str] = None+ prompt_messages: str | object = None+ prompt_messages_format: PromptMessageFormat = PromptMessageFormat.STRING+ # TODO: remove and just create it in __post_init__?+ _formatted_prompt_messages: object = field(init=False, default=None)completion: Optional[str] = Nonemodel: Optional[Models] = Noneprompt_tokens: Optional[int] = Nonecompletion_tokens: Optional[int] = None+ def __post_init__(self):+ if self.prompt_messages_format == PromptMessageFormat.STRING:+ self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}+ elif self.prompt_messages_format == PromptMessageFormat.CHATML:+ # Check if prompt_messages is already a list (indicating direct messages without "messages" key)+ if isinstance(self.prompt_messages, list):+ # Direct list of messages, add under "messages" key+ self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}+ elif "messages" in self.prompt_messages:+ # prompt_messages is a dict that includes a "messages" key+ self._formatted_prompt_messages = {"type": "chatml", **self.prompt_messages}+ else:+ logging.error("AgentOps: invalid prompt_messages format")+@dataclassclass ToolEvent(Event):@@ -60,18 +78,18 @@ class ToolEvent(Event):# Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details@dataclassclass ErrorEvent():- trigger_event_id: Optional[UUID] = None- trigger_event_type: Optional[EventType] = None+ trigger_event: Optional[Event] = Noneerror_type: Optional[str] = Nonecode: Optional[str] = Nonedetails: Optional[str] = Nonelogs: Optional[str] = None+ # event_type: str = EventType.ERROR.value # TODO: don't expose this+ event_type: str = field(init=False, default=EventType.ERROR.value)+ timestamp: str = field(default_factory=get_ISO_time)- def __init__(self, event: Event = None, **kwargs):- self.event_type: str = EventType.ERROR.value- self.timestamp = get_ISO_time()- if event:- self.trigger_event_id = event.id- self.trigger_event_type = event.event_type- for key, value in kwargs.items():- setattr(self, key, value)+ def __post_init__(self):+ if self.trigger_event:+ self.trigger_event_id = self.trigger_event.id+ self.trigger_event_type = self.trigger_event.event_type+ # TODO: remove trigger_event from serialization+ # e.g. field(repr=False, compare=False, hash=False, metadata={'serialize': False})diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 49eedb9d..2e160261 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -6,6 +6,7 @@ import loggingfrom .event import LLMEventfrom .helpers import get_ISO_time, check_call_stack_for_agent_idimport inspect+from .enums import PromptMessageFormatclass LlmTracker:@@ -29,7 +30,7 @@ class LlmTracker:def _handle_response_v0_openai(self, response, kwargs, init_timestamp):"""Handle responses for OpenAI versions <v1.0.0"""- prompt = kwargs.pop("messages", None) # pull prompt out for visibility+ prompt_messages = {"messages": kwargs.pop("messages", [])} # pull messages out for visibilitydef handle_stream_chunk(chunk):try:@@ -45,7 +46,8 @@ class LlmTracker:returns={"finish_reason": None,"content": token},agent_id=check_call_stack_for_agent_id(),- prompt=prompt,+ prompt_messages=prompt_messages,+ prompt_messages_format=PromptMessageFormat.CHATML,completion=token,model=model)@@ -64,7 +66,7 @@ class LlmTracker:except Exception as e:# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif inspect.isasyncgen(response):@@ -91,7 +93,8 @@ class LlmTracker:returns={"content":response['choices'][0]['message']['content']},agent_id=check_call_stack_for_agent_id(),- prompt=prompt,+ prompt_messages=prompt_messages,+ prompt_messages_format=PromptMessageFormat.CHATML,completion=response['choices'][0]['message']['content'],model=response['model'],prompt_tokens=response.get('usage',@@ -111,7 +114,8 @@ class LlmTracker:"content": response.choices[0].message.model_dump()},agent_id=check_call_stack_for_agent_id(),# TODO: Will need to make the completion the key for content, splat out the model dump- prompt=prompt,+ prompt_messages=prompt_messages,+ prompt_messages_format=PromptMessageFormat.CHATML,completion=response.choices[0].message.model_dump(),model=response.model,prompt_tokens=response.usage.prompt_tokens,@@ -121,7 +125,7 @@ class LlmTracker:except Exception as e:# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")return response@@ -131,7 +135,7 @@ class LlmTracker:from openai.types.chat import ChatCompletionChunkfrom openai.resources import AsyncCompletions- prompt = kwargs.pop("messages", None) # pull prompt out for visibility+ prompt_messages = {"messages": kwargs.pop("messages", [])} # pull messages out for visibilitydef handle_stream_chunk(chunk: ChatCompletionChunk):try:@@ -148,7 +152,8 @@ class LlmTracker:init_timestamp=init_timestamp,params=kwargs,agent_id=check_call_stack_for_agent_id(),- prompt=prompt,+ prompt_messages=prompt_messages,+ prompt_messages_format=PromptMessageFormat.CHATML,returns={"finish_reason": None,"content": token},model=model@@ -172,7 +177,7 @@ class LlmTracker:except Exception as e:# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif isinstance(response, Stream):@@ -207,7 +212,8 @@ class LlmTracker:# TODO: Will need to make the completion the key for content, splat out the model dump"content": response.choices[0].message.model_dump()},agent_id=check_call_stack_for_agent_id(),- prompt=prompt,+ prompt_messages=prompt_messages,+ prompt_messages_format=PromptMessageFormat.CHATML,completion=response.choices[0].message.model_dump(),model=response.model,prompt_tokens=response.usage.prompt_tokens,@@ -217,7 +223,7 @@ class LlmTracker:except Exception as e:# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")return responsediff --git a/tests/openai_handlers/test_llm_tracker_ge_1_async.py b/tests/openai_handlers/test_llm_tracker_ge_1_async.pyindex a216445c..de78fa4c 100644--- a/tests/openai_handlers/test_llm_tracker_ge_1_async.py+++ b/tests/openai_handlers/test_llm_tracker_ge_1_async.py@@ -28,7 +28,7 @@ async def call_openai_async():)print(response)- raise ValueError("This is an intentional error for testing.")+ # raise ValueError("This is an intentional error for testing.")async def main():diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/test_llm_tracker_ge_1_sync.pyindex 7fcfba19..b8e01879 100644--- a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py+++ b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py@@ -39,7 +39,7 @@ def call_openai():)print(response)- raise ValueError("This is an intentional error for testing.")+ # raise ValueError("This is an intentional error for testing.")call_openai()diff --git a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/test_llm_tracker_lt_1_sync.pyindex 70f917bf..9f33104c 100644--- a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py+++ b/tests/openai_handlers/test_llm_tracker_lt_1_sync.py@@ -27,7 +27,7 @@ def call_openai():model='gpt-3.5-turbo', messages=message, temperature=0.5)print(response)- raise ValueError("This is an intentional error for testing.")+ # raise ValueError("This is an intentional error for testing.")call_openai()