Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
v0 working
Howard Gil9ceb23e28493
diff --git a/agentops/event.py b/agentops/event.pyindex f8df6836..59bfb553 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -44,7 +44,7 @@ class LLMEvent(Event):event_type: str = EventType.LLM.valueagent_id: Optional[UUID] = Nonethread_id: Optional[UUID] = None- prompt_messages: str | object = None # TODO: remove from serialization+ prompt_messages: str | List = None # TODO: remove from serializationprompt_messages_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization# TODO: remove and just create it in __post_init__ so it can never be set by user?_formatted_prompt_messages: object = field(init=False, default=None)@@ -57,38 +57,23 @@ class LLMEvent(Event):completion_tokens: Optional[int] = Nonedef __post_init__(self):- # TODO can we just figure out if it's chatml so user doesn't have to pass anything?+ # TODO should we just figure out if it's chatml so user doesn't have to pass anything?if self.prompt_messages_format == LLMMessageFormat.STRING:self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}elif self.prompt_messages_format == LLMMessageFormat.CHATML:- # Check if prompt_messages is already a list (indicating direct messages without "messages" key)- if isinstance(self.prompt_messages, list):- # Direct list of messages, add under "messages" key- # [{'role': 'system', 'content': '...'}]- self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}- elif "messages" in self.prompt_messages:- # prompt_messages is a dict that includes a "messages" key- # {'messages': [{'role': 'system', 'content': '...'}]}- self._formatted_prompt_messages = {"type": "chatml", **self.prompt_messages}- else:- logging.error("AgentOps: invalid prompt_messages")+ self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}if self.completion_message_format == LLMMessageFormat.STRING:self._formatted_completion_message = {"type": "string", "string": self.completion_message}elif self.completion_message_format == LLMMessageFormat.CHATML:- # Check if completion_message is already a list (indicating direct messages without "messages" key)- if "message" in self.completion_message:- # completion_message is a dict that includes a "messages" key- # {'message': {'role': 'assistant', 'content': '...'}}- self._formatted_completion_message = {"type": "chatml", **self.completion_message}- elif True: # TODO- # Direct list of messages, add under "messages" key- # [{'role': 'system', 'content': '...'}]- self._formatted_completion_message = {"type": "chatml", "messages": self.completion_message}- else:- logging.error("AgentOps: invalid completion_message")-- print("\n\ncompletion:\n", self.completion, "\n\n")+ self._formatted_completion_message = {"type": "chatml", "message": self.completion_message}++ # import pprint+ # import json+ # print("\n\_formatted_prompt_messages:\n")+ # pprint(self._formatted_prompt_messages)+ # print("\n\_formatted_completion_message:\n")+ # json.dump(self._formatted_completion_message)@dataclassdiff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 0eabb85b..1b8f72ff 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -25,48 +25,43 @@ class LlmTracker:def __init__(self, client):self.client = client- self.event_stream = None+ self.completion = ""def _handle_response_v0_openai(self, response, kwargs, init_timestamp):"""Handle responses for OpenAI versions <v1.0.0"""- prompt_messages = kwargs.pop("messages") # pull messages out for visibility+ from pprint import pprint+ print("kwargs:")+ pprint(kwargs)def handle_stream_chunk(chunk):+ print("chunk:")+ pprint(chunk)try:model = chunk['model']choices = chunk['choices']token = choices[0]['delta'].get('content', '')finish_reason = choices[0]['finish_reason']+ self.completion += token- if self.event_stream == None:- self.event_stream = LLMEvent(+ if finish_reason:+ # TODO: handle case where finish_reason never gets a value? Do we record an LLMEvent in that case?+ self.client.record(LLMEvent(init_timestamp=init_timestamp,params=kwargs,- returns={"finish_reason": None,- "content": token},agent_id=check_call_stack_for_agent_id(),- prompt_messages=prompt_messages,+ prompt_messages=kwargs["messages"],prompt_messages_format=LLMMessageFormat.CHATML,- completion=token,+ completion_message={"role": "assistant", "content": self.completion},+ completion_message_format=LLMMessageFormat.CHATML,+ # TODO: cannot return response.model_dump() bc response is async generator+ returns={"finish_reason": finish_reason, "content": self.completion},model=model- )- else:- self.event_stream.returns['content'] += token- self.event_stream.completion += token-- if finish_reason:- if not self.event_stream.returns:- self.event_stream.returns = {}- self.event_stream.returns['finish_reason'] = finish_reason- # Update end_timestamp- self.event_stream.end_timestamp = get_ISO_time()- self.client.record(self.event_stream)- self.event_stream = None+ ))except Exception as e:# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif inspect.isasyncgen(response):@@ -87,45 +82,25 @@ class LlmTracker:# v0.0.0 responses are dictstry:+ print("\nresponse:")+ pprint(response.model_dump())+self.client.record(LLMEvent(init_timestamp=init_timestamp,params=kwargs,- returns={"content":- response['choices'][0]['message']['content']},agent_id=check_call_stack_for_agent_id(),- prompt_messages=prompt_messages,+ prompt_messages=kwargs["messages"],prompt_messages_format=LLMMessageFormat.CHATML,- completion=response['choices'][0]['message']['content'],- model=response['model'],- prompt_tokens=response.get('usage',- {}).get('prompt_tokens'),- completion_tokens=response.get('usage',- {}).get('completion_tokens')+ # TODO: need to coerce this into chatml+ completion_message=response['choices'][0]['message'],+ completion_message_format=LLMMessageFormat.CHATML,+ returns=response.model_dump(),+ model=response["model"]))-- except: # NOTE: Execution should never reach here. Should remove if does not break anything-- # v1.0.0+ responses are objects- try:- self.client.record(LLMEvent(- init_timestamp=init_timestamp,- params=kwargs,- returns={- "content": response.choices[0].message.model_dump()},- agent_id=check_call_stack_for_agent_id(),- # TODO: Will need to make the completion the key for content, splat out the model dump- prompt_messages=prompt_messages,- prompt_messages_format=LLMMessageFormat.CHATML,- completion=response.choices[0].message.model_dump(),- model=response.model,- prompt_tokens=response.usage.prompt_tokens,- completion_tokens=response.usage.completion_tokens- ))- # Standard response- except Exception as e:- # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths- logging.warning(- f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")+ except Exception as e:+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths+ logging.warning(+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")return response@@ -135,7 +110,11 @@ class LlmTracker:from openai.types.chat import ChatCompletionChunkfrom openai.resources import AsyncCompletions- prompt_messages = kwargs.pop("messages") # pull messages out for visibility+ from pprint import pprint+ print("kwargs:")+ pprint(kwargs)+ print("\nresponse:")+ pprint(response.model_dump())def handle_stream_chunk(chunk: ChatCompletionChunk):try:@@ -152,10 +131,11 @@ class LlmTracker:init_timestamp=init_timestamp,params=kwargs,agent_id=check_call_stack_for_agent_id(),- prompt_messages=prompt_messages,+ prompt_messages=kwargs["messages"],prompt_messages_format=LLMMessageFormat.CHATML,- returns={"finish_reason": None,- "content": token},+ completion_message=response.choices[0].message.model_dump(),+ completion_message_format=LLMMessageFormat.CHATML,+ returns=response.model_dump(),model=model)else:@@ -163,17 +143,17 @@ class LlmTracker:token = ''self.event_stream.returns['content'] += token- if finish_reason:- if not self.event_stream.returns:- self.event_stream.returns = {}- self.event_stream.returns['finish_reason'] = finish_reason- self.event_stream.returns['function_call'] = function_call- self.event_stream.returns['tool_calls'] = tool_calls- self.event_stream.returns['role'] = role- # Update end_timestamp- self.event_stream.end_timestamp = get_ISO_time()- self.client.record(self.event_stream)- self.event_stream = None+ # if finish_reason:+ # if not self.event_stream.returns:+ # self.event_stream.returns = {}+ # self.event_stream.returns['finish_reason'] = finish_reason+ # self.event_stream.returns['function_call'] = function_call+ # self.event_stream.returns['tool_calls'] = tool_calls+ # self.event_stream.returns['role'] = role+ # # Update end_timestamp+ # self.event_stream.end_timestamp = get_ISO_time()+ # self.client.record(self.event_stream)+ # self.event_stream = Noneexcept Exception as e:# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(@@ -203,35 +183,24 @@ class LlmTracker:yield chunkreturn async_generator()- # v1.0.0+ responses are objects+ # v1.0.0+ responses are objectstry:-- import json- test = response.choices[0].message- json_string = json.dumps(test)- print(json_string)- test = response.choices[0].message.model_dump()- json_string = json.dumps(test)- print(json_string)-self.client.record(LLMEvent(init_timestamp=init_timestamp,params=kwargs,- returns=response,agent_id=check_call_stack_for_agent_id(),- prompt_messages=prompt_messages,+ prompt_messages=kwargs["messages"],prompt_messages_format=LLMMessageFormat.CHATML,completion_message=response.choices[0].message.model_dump(),completion_message_format=LLMMessageFormat.CHATML,- model=response.model,- prompt_tokens=response.usage.prompt_tokens,- completion_tokens=response.usage.completion_tokens+ returns=response.model_dump(),+ model=response.model))# Standard responseexcept Exception as e:# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")return responsediff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/test_llm_tracker_ge_1_sync.pyindex b8e01879..415e0af0 100644--- a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py+++ b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py@@ -26,11 +26,11 @@ def call_openai():messages=[{"role": "system",- "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."+ "content": "You are an expert haiku writer"},{"role": "user",- "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"+ "content": "write me a haiku about devops"}],temperature=0.7,