Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
cleanup
Howard Gil1b7500ce0152
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex bb6e6735..ed9ce6cf 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -49,22 +49,17 @@ class LlmTracker:self.completion += tokenif finish_reason:- # TODO: handle case where finish_reason never gets a value? Do we record an LLMEvent in that case?self.llm_event.agent_id = check_call_stack_for_agent_id()self.llm_event.prompt_messages = kwargs["messages"]self.llm_event.prompt_messages_format = LLMMessageFormat.CHATMLself.llm_event.completion_message = {"role": "assistant", "content": self.completion}self.llm_event.completion_message_format = LLMMessageFormat.CHATML- # TODO: cannot return response.model_dump() bc response is async generatorself.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion}self.llm_event.model = modelself.llm_event.end_timestamp = get_ISO_time()self.llm_event.format_messages()self.client.record(self.llm_event)-- # self.llm_event = None- # self.completion = ""except Exception as e:self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths@@ -97,7 +92,6 @@ class LlmTracker:self.llm_event.agent_id = check_call_stack_for_agent_id()self.llm_event.prompt_messages = kwargs["messages"]self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML- # TODO: need to coerce this into chatmlself.llm_event.completion_message = response['choices'][0]['message']self.llm_event.completion_message_format = LLMMessageFormat.CHATMLself.llm_event.returns = {"content": response['choices'][0]['message']['content']}@@ -106,8 +100,6 @@ class LlmTracker:self.llm_event.format_messages()self.client.record(self.llm_event)-- # self.llm_event = Noneexcept Exception as e:self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths@@ -144,7 +136,6 @@ class LlmTracker:self.completion += tokenif finish_reason:- # TODO: handle case where finish_reason never gets a value? Do we record an LLMEvent in that case?self.llm_event.agent_id = check_call_stack_for_agent_id()self.llm_event.prompt_messages = kwargs["messages"]self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML@@ -157,9 +148,6 @@ class LlmTracker:self.llm_event.format_messages()self.client.record(self.llm_event)-- # self.llm_event = None- # self.completion = ""except Exception as e:self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths@@ -206,9 +194,6 @@ class LlmTracker:self.llm_event.format_messages()self.client.record(self.llm_event)-- # self.llm_event = None- # Standard responseexcept Exception as e:self.client.record(ErrorEvent(trigger_event=self.llm_event, details={f"{type(e).__name__}": str(e)}))# TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths