Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
WIP. Adding support for completions but needs more thought
Howard Gil79dad7316046
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 3544b612..40ffa69d 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -3,5 +3,5 @@from .client import Clientfrom .event import ActionEvent, LLMEvent, ToolEvent, ErrorEventfrom .logger import AgentOpsLogger-from .enums import Models, PromptMessageFormat+from .enums import Models, LLMMessageFormatfrom .decorators import record_functiondiff --git a/agentops/client.py b/agentops/client.pyindex 762f1505..7701397b 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -142,7 +142,7 @@ class Client(metaclass=MetaClient):except Exception as e:# TODO: add the stack trace- self.record(ErrorEvent(event=event, details={f"{type(e).__name__}": str(e)}))+ self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))# Re-raise the exceptionraise@@ -182,7 +182,7 @@ class Client(metaclass=MetaClient):except Exception as e:# TODO: add the stack trace- self.record(ErrorEvent(event=event, details={f"{type(e).__name__}": str(e)}))+ self.record(ErrorEvent(trigger_event=event, details={f"{type(e).__name__}": str(e)}))# Re-raise the exceptionraisediff --git a/agentops/enums.py b/agentops/enums.pyindex ac0d087b..8f704968 100644--- a/agentops/enums.py+++ b/agentops/enums.py@@ -29,6 +29,6 @@ class Models(Enum):TEXT_EMBEDDING_ADA_002 = "text-embedding-ada-002"-class PromptMessageFormat(Enum):+class LLMMessageFormat(Enum):STRING = "string"CHATML = "chatml"diff --git a/agentops/event.py b/agentops/event.pyindex 34c395c0..f8df6836 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -8,7 +8,7 @@ Data Class:from dataclasses import dataclass, fieldfrom typing import List, Optionalfrom .helpers import get_ISO_time-from .enums import EventType, Models, PromptMessageFormat+from .enums import EventType, Models, LLMMessageFormatfrom uuid import UUID, uuid4import logging@@ -21,6 +21,7 @@ class Event:init_timestamp: Optional[str] = field(default_factory=get_ISO_time)end_timestamp: str = field(default_factory=get_ISO_time)id: UUID = field(default_factory=uuid4)+ # TODO: has_been_recorded: bool = False@dataclass@@ -43,28 +44,51 @@ class LLMEvent(Event):event_type: str = EventType.LLM.valueagent_id: Optional[UUID] = Nonethread_id: Optional[UUID] = None- prompt_messages: str | object = None- prompt_messages_format: PromptMessageFormat = PromptMessageFormat.STRING- # TODO: remove and just create it in __post_init__?+ prompt_messages: str | object = None # TODO: remove from serialization+ prompt_messages_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization+ # TODO: remove and just create it in __post_init__ so it can never be set by user?_formatted_prompt_messages: object = field(init=False, default=None)- completion: Optional[str] = None+ completion_message: str | object = None # TODO: remove from serialization+ completion_message_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization+ # TODO: remove and just create it in __post_init__ so it can never be set by user?+ _formatted_completion_message: object = field(init=False, default=None)model: Optional[Models] = Noneprompt_tokens: Optional[int] = Nonecompletion_tokens: Optional[int] = Nonedef __post_init__(self):- if self.prompt_messages_format == PromptMessageFormat.STRING:+ # TODO can we just figure out if it's chatml so user doesn't have to pass anything?+ if self.prompt_messages_format == LLMMessageFormat.STRING:self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}- elif self.prompt_messages_format == PromptMessageFormat.CHATML:+ elif self.prompt_messages_format == LLMMessageFormat.CHATML:# Check if prompt_messages is already a list (indicating direct messages without "messages" key)if isinstance(self.prompt_messages, list):# Direct list of messages, add under "messages" key+ # [{'role': 'system', 'content': '...'}]self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}elif "messages" in self.prompt_messages:# prompt_messages is a dict that includes a "messages" key+ # {'messages': [{'role': 'system', 'content': '...'}]}self._formatted_prompt_messages = {"type": "chatml", **self.prompt_messages}else:- logging.error("AgentOps: invalid prompt_messages format")+ logging.error("AgentOps: invalid prompt_messages")++ if self.completion_message_format == LLMMessageFormat.STRING:+ self._formatted_completion_message = {"type": "string", "string": self.completion_message}+ elif self.completion_message_format == LLMMessageFormat.CHATML:+ # Check if completion_message is already a list (indicating direct messages without "messages" key)+ if "message" in self.completion_message:+ # completion_message is a dict that includes a "messages" key+ # {'message': {'role': 'assistant', 'content': '...'}}+ self._formatted_completion_message = {"type": "chatml", **self.completion_message}+ elif True: # TODO+ # Direct list of messages, add under "messages" key+ # [{'role': 'system', 'content': '...'}]+ self._formatted_completion_message = {"type": "chatml", "messages": self.completion_message}+ else:+ logging.error("AgentOps: invalid completion_message")++ print("\n\ncompletion:\n", self.completion, "\n\n")@dataclass@@ -78,7 +102,7 @@ class ToolEvent(Event):# Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details@dataclassclass ErrorEvent():- trigger_event: Optional[Event] = None+ trigger_event: Optional[Event] = None # TODO: remove from serialization?error_type: Optional[str] = Nonecode: Optional[str] = Nonedetails: Optional[str] = Nonediff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 2e160261..0eabb85b 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -6,7 +6,7 @@ import loggingfrom .event import LLMEventfrom .helpers import get_ISO_time, check_call_stack_for_agent_idimport inspect-from .enums import PromptMessageFormat+from .enums import LLMMessageFormatclass LlmTracker:@@ -30,7 +30,7 @@ class LlmTracker:def _handle_response_v0_openai(self, response, kwargs, init_timestamp):"""Handle responses for OpenAI versions <v1.0.0"""- prompt_messages = {"messages": kwargs.pop("messages", [])} # pull messages out for visibility+ prompt_messages = kwargs.pop("messages") # pull messages out for visibilitydef handle_stream_chunk(chunk):try:@@ -47,7 +47,7 @@ class LlmTracker:"content": token},agent_id=check_call_stack_for_agent_id(),prompt_messages=prompt_messages,- prompt_messages_format=PromptMessageFormat.CHATML,+ prompt_messages_format=LLMMessageFormat.CHATML,completion=token,model=model)@@ -94,7 +94,7 @@ class LlmTracker:response['choices'][0]['message']['content']},agent_id=check_call_stack_for_agent_id(),prompt_messages=prompt_messages,- prompt_messages_format=PromptMessageFormat.CHATML,+ prompt_messages_format=LLMMessageFormat.CHATML,completion=response['choices'][0]['message']['content'],model=response['model'],prompt_tokens=response.get('usage',@@ -115,7 +115,7 @@ class LlmTracker:agent_id=check_call_stack_for_agent_id(),# TODO: Will need to make the completion the key for content, splat out the model dumpprompt_messages=prompt_messages,- prompt_messages_format=PromptMessageFormat.CHATML,+ prompt_messages_format=LLMMessageFormat.CHATML,completion=response.choices[0].message.model_dump(),model=response.model,prompt_tokens=response.usage.prompt_tokens,@@ -135,7 +135,7 @@ class LlmTracker:from openai.types.chat import ChatCompletionChunkfrom openai.resources import AsyncCompletions- prompt_messages = {"messages": kwargs.pop("messages", [])} # pull messages out for visibility+ prompt_messages = kwargs.pop("messages") # pull messages out for visibilitydef handle_stream_chunk(chunk: ChatCompletionChunk):try:@@ -153,7 +153,7 @@ class LlmTracker:params=kwargs,agent_id=check_call_stack_for_agent_id(),prompt_messages=prompt_messages,- prompt_messages_format=PromptMessageFormat.CHATML,+ prompt_messages_format=LLMMessageFormat.CHATML,returns={"finish_reason": None,"content": token},model=model@@ -205,16 +205,24 @@ class LlmTracker:# v1.0.0+ responses are objectstry:++ import json+ test = response.choices[0].message+ json_string = json.dumps(test)+ print(json_string)+ test = response.choices[0].message.model_dump()+ json_string = json.dumps(test)+ print(json_string)+self.client.record(LLMEvent(init_timestamp=init_timestamp,params=kwargs,- returns={- # TODO: Will need to make the completion the key for content, splat out the model dump- "content": response.choices[0].message.model_dump()},+ returns=response,agent_id=check_call_stack_for_agent_id(),prompt_messages=prompt_messages,- prompt_messages_format=PromptMessageFormat.CHATML,- completion=response.choices[0].message.model_dump(),+ prompt_messages_format=LLMMessageFormat.CHATML,+ completion_message=response.choices[0].message.model_dump(),+ completion_message_format=LLMMessageFormat.CHATML,model=response.model,prompt_tokens=response.usage.prompt_tokens,completion_tokens=response.usage.completion_tokens