Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Fixing LLMEvent returns and streaming logic (#181)
Howard Gil562a1ac76249
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 539bd56a..507b8504 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -7,6 +7,7 @@ from .log_config import loggerfrom .event import LLMEvent, ErrorEventfrom .helpers import get_ISO_time, check_call_stack_for_agent_idimport inspect+import pprintclass LlmTracker:@@ -26,44 +27,51 @@ class LlmTracker:def __init__(self, client):self.client = client- self.completion = ""- self.llm_event: LLMEvent = Nonedef _handle_response_v0_openai(self, response, kwargs, init_timestamp):"""Handle responses for OpenAI versions <v1.0.0"""- self.completion = ""- self.llm_event = None+ self.llm_event = LLMEvent(+ init_timestamp=init_timestamp,+ params=kwargs+ )def handle_stream_chunk(chunk):- self.llm_event = LLMEvent(- init_timestamp=init_timestamp,- params=kwargs- )+ # NOTE: prompt/completion usage not returned in response when streaming+ # We take the first ChatCompletionChunk and accumulate the deltas from all subsequent chunks to build one full chat completion+ if self.llm_event.returns == None:+ self.llm_event.returns = chunktry:- # NOTE: prompt/completion usage not returned in response when streaming- model = chunk['model']- choices = chunk['choices']- token = choices[0]['delta'].get('content', '')- finish_reason = choices[0]['finish_reason']- if token:- self.completion += token-- if finish_reason:- self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt = kwargs["messages"]- self.llm_event.completion = {"role": "assistant", "content": self.completion}- self.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion}- self.llm_event.model = model+ accumulated_delta = self.llm_event.returns['choices'][0]['delta']+ self.llm_event.agent_id = check_call_stack_for_agent_id()+ self.llm_event.model = chunk['model']+ self.llm_event.prompt = kwargs["messages"]+ choice = chunk['choices'][0] # NOTE: We assume for completion only choices[0] is relevant++ if choice['delta'].get('content'):+ accumulated_delta['content'] += choice['delta'].content++ if choice['delta'].get('role'):+ accumulated_delta['role'] = choice['delta'].get('role')++ if choice['finish_reason']:+ # Streaming is done. Record LLMEvent+ self.llm_event.returns.choices[0]['finish_reason'] = choice['finish_reason']+ self.llm_event.completion = {+ "role": accumulated_delta['role'], "content": accumulated_delta['content']}self.llm_event.end_timestamp = get_ISO_time()self.client.record(self.llm_event)except Exception as e:self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e))- # TODO: This error is specific to only one path of failure. Should be more generic or have different logger for different paths+ kwargs_str = pprint.pformat(kwargs)+ chunk = pprint.pformat(chunk)logger.warning(- f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"🖇 AgentOps: Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n"+ f"chunk:\n {chunk}\n"+ f"kwargs:\n {kwargs_str}\n"+ )# if the response is a generator, decorate the generatorif inspect.isasyncgen(response):@@ -84,7 +92,8 @@ class LlmTracker:self.llm_event = LLMEvent(init_timestamp=init_timestamp,- params=kwargs+ params=kwargs,+ returns=response)# v0.0.0 responses are dictstry:@@ -93,16 +102,19 @@ class LlmTracker:self.llm_event.prompt_tokens = response['usage']['prompt_tokens']self.llm_event.completion = {"role": "assistant", "content": response['choices'][0]['message']['content']}self.llm_event.completion_tokens = response['usage']['completion_tokens']- self.llm_event.returns = {"content": response['choices'][0]['message']['content']}self.llm_event.model = response["model"]self.llm_event.end_timestamp = get_ISO_time()self.client.record(self.llm_event)except Exception as e:self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e))- # TODO: This error is specific to only one path of failure. Should be more generic or have different logger for different paths+ kwargs_str = pprint.pformat(kwargs)+ response = pprint.pformat(response)logger.warning(- f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"🖇 AgentOps: Unable to parse response for LLM call. Skipping upload to AgentOps\n"+ f"response:\n {response}\n"+ f"kwargs:\n {kwargs_str}\n"+ )return response@@ -112,43 +124,53 @@ class LlmTracker:from openai.types.chat import ChatCompletionChunkfrom openai.resources import AsyncCompletions- self.completion = ""- self.llm_event = None+ self.llm_event = LLMEvent(+ init_timestamp=init_timestamp,+ params=kwargs+ )def handle_stream_chunk(chunk: ChatCompletionChunk):-- self.llm_event = LLMEvent(- init_timestamp=init_timestamp,- params=kwargs- )+ # NOTE: prompt/completion usage not returned in response when streaming+ # We take the first ChatCompletionChunk and accumulate the deltas from all subsequent chunks to build one full chat completion+ if self.llm_event.returns == None:+ self.llm_event.returns = chunktry:- # NOTE: prompt/completion usage not returned in response when streaming- model = chunk.model- choices = chunk.choices- token = choices[0].delta.content- finish_reason = choices[0].finish_reason- function_call = choices[0].delta.function_call- tool_calls = choices[0].delta.tool_calls- role = choices[0].delta.role- if token:- self.completion += token-- if finish_reason:- self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt = kwargs["messages"]- self.llm_event.completion = {"role": "assistant", "content": self.completion}- self.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion,- "function_call": function_call, "tool_calls": tool_calls, "role": role}- self.llm_event.model = model+ accumulated_delta = self.llm_event.returns.choices[0].delta+ self.llm_event.agent_id = check_call_stack_for_agent_id()+ self.llm_event.model = chunk.model+ self.llm_event.prompt = kwargs["messages"]+ choice = chunk.choices[0] # NOTE: We assume for completion only choices[0] is relevant++ if choice.delta.content:+ accumulated_delta.content += choice.delta.content++ if choice.delta.role:+ accumulated_delta.role = choice.delta.role++ if choice.delta.tool_calls:+ accumulated_delta.tool_calls = choice.delta.tool_calls++ if choice.delta.function_call:+ accumulated_delta.function_call = choice.delta.function_call++ if choice.finish_reason:+ # Streaming is done. Record LLMEvent+ self.llm_event.returns.choices[0].finish_reason = choice.finish_reason+ self.llm_event.completion = {"role": accumulated_delta.role, "content": accumulated_delta.content,+ "function_call": accumulated_delta.function_call, "tool_calls": accumulated_delta.tool_calls}self.llm_event.end_timestamp = get_ISO_time()self.client.record(self.llm_event)except Exception as e:self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e))- # TODO: This error is specific to only one path of failure. Should be more generic or have different logger for different paths+ kwargs_str = pprint.pformat(kwargs)+ chunk = pprint.pformat(chunk)logger.warning(- f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"🖇 AgentOps: Unable to parse a chunk for LLM call. Skipping upload to AgentOps\n"+ f"chunk:\n {chunk}\n"+ f"kwargs:\n {kwargs_str}\n"+ )# if the response is a generator, decorate the generatorif isinstance(response, Stream):@@ -180,20 +202,24 @@ class LlmTracker:)# v1.0.0+ responses are objectstry:+ self.llm_event.returns = response.model_dump()self.llm_event.agent_id = check_call_stack_for_agent_id()self.llm_event.prompt = kwargs["messages"]self.llm_event.prompt_tokens = response.usage.prompt_tokensself.llm_event.completion = response.choices[0].message.model_dump()self.llm_event.completion_tokens = response.usage.completion_tokens- self.llm_event.returns = response.model_dump()self.llm_event.model = response.modelself.client.record(self.llm_event)except Exception as e:self.client.record(ErrorEvent(trigger_event=self.llm_event, exception=e))- # TODO: This error is specific to only one path of failure. Should be more generic or have different logger for different paths+ kwargs_str = pprint.pformat(kwargs)+ response = pprint.pformat(response)logger.warning(- f"🖇 AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"🖇 AgentOps: Unable to parse response for LLM call. Skipping upload to AgentOps\n"+ f"response:\n {response}\n"+ f"kwargs:\n {kwargs_str}\n"+ )return response