Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Tested all routes
Howard Gil628aea30c471
diff --git a/agentops/event.py b/agentops/event.pyindex 59bfb553..0f192f99 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -56,24 +56,23 @@ class LLMEvent(Event):prompt_tokens: Optional[int] = Nonecompletion_tokens: Optional[int] = None- def __post_init__(self):- # TODO should we just figure out if it's chatml so user doesn't have to pass anything?- if self.prompt_messages_format == LLMMessageFormat.STRING:- self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}- elif self.prompt_messages_format == LLMMessageFormat.CHATML:- self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}-- if self.completion_message_format == LLMMessageFormat.STRING:- self._formatted_completion_message = {"type": "string", "string": self.completion_message}- elif self.completion_message_format == LLMMessageFormat.CHATML:- self._formatted_completion_message = {"type": "chatml", "message": self.completion_message}+ def format_messages(self):+ if self.prompt_messages:+ # TODO should we just figure out if it's chatml so user doesn't have to pass anything?+ if self.prompt_messages_format == LLMMessageFormat.STRING:+ self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}+ elif self.prompt_messages_format == LLMMessageFormat.CHATML:+ self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}++ if self.completion_message:+ if self.completion_message_format == LLMMessageFormat.STRING:+ self._formatted_completion_message = {"type": "string", "string": self.completion_message}+ elif self.completion_message_format == LLMMessageFormat.CHATML:+ self._formatted_completion_message = {"type": "chatml", "message": self.completion_message}- # import pprint- # import json- # print("\n\_formatted_prompt_messages:\n")- # pprint(self._formatted_prompt_messages)- # print("\n\_formatted_completion_message:\n")- # json.dump(self._formatted_completion_message)+ def __post_init__(self):+ # format if prompt/completion messages were passed when LLMEvent was created+ self.format_messages()@dataclass@@ -92,11 +91,10 @@ class ErrorEvent():code: Optional[str] = Nonedetails: Optional[str] = Nonelogs: Optional[str] = None- # event_type: str = EventType.ERROR.value # TODO: don't expose this- event_type: str = field(init=False, default=EventType.ERROR.value)timestamp: str = field(default_factory=get_ISO_time)def __post_init__(self):+ self.event_type = EventType.ERROR.valueif self.trigger_event:self.trigger_event_id = self.trigger_event.idself.trigger_event_type = self.trigger_event.event_typediff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 1b8f72ff..bb6e6735 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -3,7 +3,7 @@ import sysfrom importlib import import_modulefrom packaging.version import parseimport logging-from .event import LLMEvent+from .event import LLMEvent, ErrorEventfrom .helpers import get_ISO_time, check_call_stack_for_agent_idimport inspectfrom .enums import LLMMessageFormat@@ -26,39 +26,47 @@ class LlmTracker:def __init__(self, client):self.client = clientself.completion = ""+ self.llm_event: LLMEvent = Nonedef _handle_response_v0_openai(self, response, kwargs, init_timestamp):"""Handle responses for OpenAI versions <v1.0.0"""- from pprint import pprint- print("kwargs:")- pprint(kwargs)+ self.completion = ""+ self.llm_event = Nonedef handle_stream_chunk(chunk):- print("chunk:")- pprint(chunk)+ self.llm_event = LLMEvent(+ init_timestamp=init_timestamp,+ params=kwargs+ )+try:model = chunk['model']choices = chunk['choices']token = choices[0]['delta'].get('content', '')finish_reason = choices[0]['finish_reason']- self.completion += token+ if token:+ self.completion += tokenif finish_reason:# TODO: handle case where finish_reason never gets a value? Do we record an LLMEvent in that case?- self.client.record(LLMEvent(- init_timestamp=init_timestamp,- params=kwargs,- agent_id=check_call_stack_for_agent_id(),- prompt_messages=kwargs["messages"],- prompt_messages_format=LLMMessageFormat.CHATML,- completion_message={"role": "assistant", "content": self.completion},- completion_message_format=LLMMessageFormat.CHATML,- # TODO: cannot return response.model_dump() bc response is async generator- returns={"finish_reason": finish_reason, "content": self.completion},- model=model- ))+ self.llm_event.agent_id = check_call_stack_for_agent_id()+ self.llm_event.prompt_messages = kwargs["messages"]+ self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML+ self.llm_event.completion_message = {"role": "assistant", "content": self.completion}+ self.llm_event.completion_message_format = LLMMessageFormat.CHATML+ # TODO: cannot return response.model_dump() bc response is async generator+ self.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion}+ self.llm_event.model = model+ self.llm_event.end_timestamp = get_ISO_time()+ self.llm_event.format_messages()++ self.client.record(self.llm_event)++ # self.llm_event = None+ # self.completion = ""except Exception as e:+ self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")@@ -80,24 +88,28 @@ class LlmTracker:yield chunkreturn generator()+ self.llm_event = LLMEvent(+ init_timestamp=init_timestamp,+ params=kwargs+ )# v0.0.0 responses are dictstry:- print("\nresponse:")- pprint(response.model_dump())-- self.client.record(LLMEvent(- init_timestamp=init_timestamp,- params=kwargs,- agent_id=check_call_stack_for_agent_id(),- prompt_messages=kwargs["messages"],- prompt_messages_format=LLMMessageFormat.CHATML,- # TODO: need to coerce this into chatml- completion_message=response['choices'][0]['message'],- completion_message_format=LLMMessageFormat.CHATML,- returns=response.model_dump(),- model=response["model"]- ))+ self.llm_event.agent_id = check_call_stack_for_agent_id()+ self.llm_event.prompt_messages = kwargs["messages"]+ self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML+ # TODO: need to coerce this into chatml+ self.llm_event.completion_message = response['choices'][0]['message']+ self.llm_event.completion_message_format = LLMMessageFormat.CHATML+ self.llm_event.returns = {"content": response['choices'][0]['message']['content']}+ self.llm_event.model = response["model"]+ self.llm_event.end_timestamp = get_ISO_time()+ self.llm_event.format_messages()++ self.client.record(self.llm_event)++ # self.llm_event = Noneexcept Exception as e:+ self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")@@ -110,13 +122,16 @@ class LlmTracker:from openai.types.chat import ChatCompletionChunkfrom openai.resources import AsyncCompletions- from pprint import pprint- print("kwargs:")- pprint(kwargs)- print("\nresponse:")- pprint(response.model_dump())+ self.completion = ""+ self.llm_event = Nonedef handle_stream_chunk(chunk: ChatCompletionChunk):++ self.llm_event = LLMEvent(+ init_timestamp=init_timestamp,+ params=kwargs+ )+try:model = chunk.modelchoices = chunk.choices@@ -125,39 +140,31 @@ class LlmTracker:function_call = choices[0].delta.function_calltool_calls = choices[0].delta.tool_callsrole = choices[0].delta.role+ if token:+ self.completion += token- if self.event_stream == None:- self.event_stream = LLMEvent(- init_timestamp=init_timestamp,- params=kwargs,- agent_id=check_call_stack_for_agent_id(),- prompt_messages=kwargs["messages"],- prompt_messages_format=LLMMessageFormat.CHATML,- completion_message=response.choices[0].message.model_dump(),- completion_message_format=LLMMessageFormat.CHATML,- returns=response.model_dump(),- model=model- )- else:- if token == None:- token = ''- self.event_stream.returns['content'] += token-- # if finish_reason:- # if not self.event_stream.returns:- # self.event_stream.returns = {}- # self.event_stream.returns['finish_reason'] = finish_reason- # self.event_stream.returns['function_call'] = function_call- # self.event_stream.returns['tool_calls'] = tool_calls- # self.event_stream.returns['role'] = role- # # Update end_timestamp- # self.event_stream.end_timestamp = get_ISO_time()- # self.client.record(self.event_stream)- # self.event_stream = None+ if finish_reason:+ # TODO: handle case where finish_reason never gets a value? Do we record an LLMEvent in that case?+ self.llm_event.agent_id = check_call_stack_for_agent_id()+ self.llm_event.prompt_messages = kwargs["messages"]+ self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML+ self.llm_event.completion_message = {"role": "assistant", "content": self.completion}+ self.llm_event.completion_message_format = LLMMessageFormat.CHATML+ self.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion,+ "function_call": function_call, "tool_calls": tool_calls, "role": role}+ self.llm_event.model = model+ self.llm_event.end_timestamp = get_ISO_time()+ self.llm_event.format_messages()++ self.client.record(self.llm_event)++ # self.llm_event = None+ # self.completion = ""except Exception as e:+ self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif isinstance(response, Stream):@@ -183,21 +190,27 @@ class LlmTracker:yield chunkreturn async_generator()- # v1.0.0+ responses are objects+ self.llm_event = LLMEvent(+ init_timestamp=init_timestamp,+ params=kwargs+ )+ # v1.0.0+ responses are objectstry:- self.client.record(LLMEvent(- init_timestamp=init_timestamp,- params=kwargs,- agent_id=check_call_stack_for_agent_id(),- prompt_messages=kwargs["messages"],- prompt_messages_format=LLMMessageFormat.CHATML,- completion_message=response.choices[0].message.model_dump(),- completion_message_format=LLMMessageFormat.CHATML,- returns=response.model_dump(),- model=response.model- ))+ self.llm_event.agent_id = check_call_stack_for_agent_id()+ self.llm_event.prompt_messages = kwargs["messages"]+ self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML+ self.llm_event.completion_message = response.choices[0].message.model_dump()+ self.llm_event.completion_message_format = LLMMessageFormat.CHATML+ self.llm_event.returns = response.model_dump()+ self.llm_event.model = response.model+ self.llm_event.format_messages()++ self.client.record(self.llm_event)++ # self.llm_event = None# Standard responseexcept Exception as e:+ self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")