Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

v0 working

Howard Gil9ceb23e28493
diff --git a/agentops/event.py b/agentops/event.py
index f8df6836..59bfb553 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -44,7 +44,7 @@ class LLMEvent(Event):
event_type: str = EventType.LLM.value
agent_id: Optional[UUID] = None
thread_id: Optional[UUID] = None
- prompt_messages: str | object = None # TODO: remove from serialization
+ prompt_messages: str | List = None # TODO: remove from serialization
prompt_messages_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization
# TODO: remove and just create it in __post_init__ so it can never be set by user?
_formatted_prompt_messages: object = field(init=False, default=None)
@@ -57,38 +57,23 @@ class LLMEvent(Event):
completion_tokens: Optional[int] = None
def __post_init__(self):
- # TODO can we just figure out if it's chatml so user doesn't have to pass anything?
+ # TODO should we just figure out if it's chatml so user doesn't have to pass anything?
if self.prompt_messages_format == LLMMessageFormat.STRING:
self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}
elif self.prompt_messages_format == LLMMessageFormat.CHATML:
- # Check if prompt_messages is already a list (indicating direct messages without "messages" key)
- if isinstance(self.prompt_messages, list):
- # Direct list of messages, add under "messages" key
- # [{'role': 'system', 'content': '...'}]
- self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}
- elif "messages" in self.prompt_messages:
- # prompt_messages is a dict that includes a "messages" key
- # {'messages': [{'role': 'system', 'content': '...'}]}
- self._formatted_prompt_messages = {"type": "chatml", **self.prompt_messages}
- else:
- logging.error("AgentOps: invalid prompt_messages")
+ self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}
if self.completion_message_format == LLMMessageFormat.STRING:
self._formatted_completion_message = {"type": "string", "string": self.completion_message}
elif self.completion_message_format == LLMMessageFormat.CHATML:
- # Check if completion_message is already a list (indicating direct messages without "messages" key)
- if "message" in self.completion_message:
- # completion_message is a dict that includes a "messages" key
- # {'message': {'role': 'assistant', 'content': '...'}}
- self._formatted_completion_message = {"type": "chatml", **self.completion_message}
- elif True: # TODO
- # Direct list of messages, add under "messages" key
- # [{'role': 'system', 'content': '...'}]
- self._formatted_completion_message = {"type": "chatml", "messages": self.completion_message}
- else:
- logging.error("AgentOps: invalid completion_message")
-
- print("\n\ncompletion:\n", self.completion, "\n\n")
+ self._formatted_completion_message = {"type": "chatml", "message": self.completion_message}
+
+ # import pprint
+ # import json
+ # print("\n\_formatted_prompt_messages:\n")
+ # pprint(self._formatted_prompt_messages)
+ # print("\n\_formatted_completion_message:\n")
+ # json.dump(self._formatted_completion_message)
@dataclass
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
index 0eabb85b..1b8f72ff 100644
--- a/agentops/llm_tracker.py
+++ b/agentops/llm_tracker.py
@@ -25,48 +25,43 @@ class LlmTracker:
def __init__(self, client):
self.client = client
- self.event_stream = None
+ self.completion = ""
def _handle_response_v0_openai(self, response, kwargs, init_timestamp):
"""Handle responses for OpenAI versions <v1.0.0"""
- prompt_messages = kwargs.pop("messages") # pull messages out for visibility
+ from pprint import pprint
+ print("kwargs:")
+ pprint(kwargs)
def handle_stream_chunk(chunk):
+ print("chunk:")
+ pprint(chunk)
try:
model = chunk['model']
choices = chunk['choices']
token = choices[0]['delta'].get('content', '')
finish_reason = choices[0]['finish_reason']
+ self.completion += token
- if self.event_stream == None:
- self.event_stream = LLMEvent(
+ if finish_reason:
+ # TODO: handle case where finish_reason never gets a value? Do we record an LLMEvent in that case?
+ self.client.record(LLMEvent(
init_timestamp=init_timestamp,
params=kwargs,
- returns={"finish_reason": None,
- "content": token},
agent_id=check_call_stack_for_agent_id(),
- prompt_messages=prompt_messages,
+ prompt_messages=kwargs["messages"],
prompt_messages_format=LLMMessageFormat.CHATML,
- completion=token,
+ completion_message={"role": "assistant", "content": self.completion},
+ completion_message_format=LLMMessageFormat.CHATML,
+ # TODO: cannot return response.model_dump() bc response is async generator
+ returns={"finish_reason": finish_reason, "content": self.completion},
model=model
- )
- else:
- self.event_stream.returns['content'] += token
- self.event_stream.completion += token
-
- if finish_reason:
- if not self.event_stream.returns:
- self.event_stream.returns = {}
- self.event_stream.returns['finish_reason'] = finish_reason
- # Update end_timestamp
- self.event_stream.end_timestamp = get_ISO_time()
- self.client.record(self.event_stream)
- self.event_stream = None
+ ))
except Exception as e:
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
if inspect.isasyncgen(response):
@@ -87,45 +82,25 @@ class LlmTracker:
# v0.0.0 responses are dicts
try:
+ print("\nresponse:")
+ pprint(response.model_dump())
+
self.client.record(LLMEvent(
init_timestamp=init_timestamp,
params=kwargs,
- returns={"content":
- response['choices'][0]['message']['content']},
agent_id=check_call_stack_for_agent_id(),
- prompt_messages=prompt_messages,
+ prompt_messages=kwargs["messages"],
prompt_messages_format=LLMMessageFormat.CHATML,
- completion=response['choices'][0]['message']['content'],
- model=response['model'],
- prompt_tokens=response.get('usage',
- {}).get('prompt_tokens'),
- completion_tokens=response.get('usage',
- {}).get('completion_tokens')
+ # TODO: need to coerce this into chatml
+ completion_message=response['choices'][0]['message'],
+ completion_message_format=LLMMessageFormat.CHATML,
+ returns=response.model_dump(),
+ model=response["model"]
))
-
- except: # NOTE: Execution should never reach here. Should remove if does not break anything
-
- # v1.0.0+ responses are objects
- try:
- self.client.record(LLMEvent(
- init_timestamp=init_timestamp,
- params=kwargs,
- returns={
- "content": response.choices[0].message.model_dump()},
- agent_id=check_call_stack_for_agent_id(),
- # TODO: Will need to make the completion the key for content, splat out the model dump
- prompt_messages=prompt_messages,
- prompt_messages_format=LLMMessageFormat.CHATML,
- completion=response.choices[0].message.model_dump(),
- model=response.model,
- prompt_tokens=response.usage.prompt_tokens,
- completion_tokens=response.usage.completion_tokens
- ))
- # Standard response
- except Exception as e:
- # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
- logging.warning(
- f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")
+ except Exception as e:
+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
+ logging.warning(
+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
return response
@@ -135,7 +110,11 @@ class LlmTracker:
from openai.types.chat import ChatCompletionChunk
from openai.resources import AsyncCompletions
- prompt_messages = kwargs.pop("messages") # pull messages out for visibility
+ from pprint import pprint
+ print("kwargs:")
+ pprint(kwargs)
+ print("\nresponse:")
+ pprint(response.model_dump())
def handle_stream_chunk(chunk: ChatCompletionChunk):
try:
@@ -152,10 +131,11 @@ class LlmTracker:
init_timestamp=init_timestamp,
params=kwargs,
agent_id=check_call_stack_for_agent_id(),
- prompt_messages=prompt_messages,
+ prompt_messages=kwargs["messages"],
prompt_messages_format=LLMMessageFormat.CHATML,
- returns={"finish_reason": None,
- "content": token},
+ completion_message=response.choices[0].message.model_dump(),
+ completion_message_format=LLMMessageFormat.CHATML,
+ returns=response.model_dump(),
model=model
)
else:
@@ -163,17 +143,17 @@ class LlmTracker:
token = ''
self.event_stream.returns['content'] += token
- if finish_reason:
- if not self.event_stream.returns:
- self.event_stream.returns = {}
- self.event_stream.returns['finish_reason'] = finish_reason
- self.event_stream.returns['function_call'] = function_call
- self.event_stream.returns['tool_calls'] = tool_calls
- self.event_stream.returns['role'] = role
- # Update end_timestamp
- self.event_stream.end_timestamp = get_ISO_time()
- self.client.record(self.event_stream)
- self.event_stream = None
+ # if finish_reason:
+ # if not self.event_stream.returns:
+ # self.event_stream.returns = {}
+ # self.event_stream.returns['finish_reason'] = finish_reason
+ # self.event_stream.returns['function_call'] = function_call
+ # self.event_stream.returns['tool_calls'] = tool_calls
+ # self.event_stream.returns['role'] = role
+ # # Update end_timestamp
+ # self.event_stream.end_timestamp = get_ISO_time()
+ # self.client.record(self.event_stream)
+ # self.event_stream = None
except Exception as e:
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
@@ -203,35 +183,24 @@ class LlmTracker:
yield chunk
return async_generator()
- # v1.0.0+ responses are objects
+ # v1.0.0+ responses are objects
try:
-
- import json
- test = response.choices[0].message
- json_string = json.dumps(test)
- print(json_string)
- test = response.choices[0].message.model_dump()
- json_string = json.dumps(test)
- print(json_string)
-
self.client.record(LLMEvent(
init_timestamp=init_timestamp,
params=kwargs,
- returns=response,
agent_id=check_call_stack_for_agent_id(),
- prompt_messages=prompt_messages,
+ prompt_messages=kwargs["messages"],
prompt_messages_format=LLMMessageFormat.CHATML,
completion_message=response.choices[0].message.model_dump(),
completion_message_format=LLMMessageFormat.CHATML,
- model=response.model,
- prompt_tokens=response.usage.prompt_tokens,
- completion_tokens=response.usage.completion_tokens
+ returns=response.model_dump(),
+ model=response.model
))
# Standard response
except Exception as e:
# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps: Unable to parse a chunk for LLM call {prompt_messages} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
return response
diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py
index b8e01879..415e0af0 100644
--- a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py
+++ b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py
@@ -26,11 +26,11 @@ def call_openai():
messages=[
{
"role": "system",
- "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."
+ "content": "You are an expert haiku writer"
},
{
"role": "user",
- "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"
+ "content": "write me a haiku about devops"
}
],
temperature=0.7,