Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
WIP. Updated llm_tracker to use LLMEvent
Howard Gil7006a2f0102e
diff --git a/agentops/event.py b/agentops/event.pyindex 576c0c8d..57456f63 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -16,6 +16,7 @@ from uuid import UUIDclass Event:event_type: str # EventType.ENUM.valuetags: Optional[List[str]] = None+ params: Optional[str] = None # TODO: We should add params to all tables in supabase, right now it's just LLMEventinit_timestamp: Optional[str] = field(default_factory=get_ISO_time)end_timestamp: str = field(default_factory=get_ISO_time)@@ -48,9 +49,12 @@ class ErrorEvent(Event):@dataclassclass LLMEvent(Event):event_type: str = EventType.LLM.value+ params: Optional[str] = None+ agent_id: Optional[UUID] = Nonethread_id: Optional[UUID] = None- model: Optional[Models] = Noneprompt: Optional[str] = None+ completion: Optional[str] = None+ model: Optional[Models] = Noneprompt_tokens: Optional[int] = Nonecompletion_tokens: Optional[int] = Nonediff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 95addc21..9a63f417 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -3,7 +3,7 @@ import sysfrom importlib import import_modulefrom packaging.version import parseimport logging-from .event import Event+from .event import LLMEventfrom .helpers import get_ISO_time, check_call_stack_for_agent_id@@ -36,18 +36,14 @@ class LlmTracker:finish_reason = choices[0]['finish_reason']if self.event_stream == None:- agent_id = check_call_stack_for_agent_id()- self.event_stream = Event(- event_type='openai stream',- agent_id=agent_id,- params=kwargs,- result='Success',- returns={"finish_reason": None,- "content": token},- action_type='llm',- model=model,+ self.event_stream = LLMEvent(+ init_timestamp=init_timestamp,+ params=kwargs["invocation_params"],+ agent_id=check_call_stack_for_agent_id(),prompt=kwargs["messages"],- init_timestamp=init_timestamp+ completion={"finish_reason": None,+ "content": token},+ model=model)else:self.event_stream.returns['content'] += token@@ -83,17 +79,14 @@ class LlmTracker:# v0.0.0 responses are dictstry:- agent_id = check_call_stack_for_agent_id()- self.client.record(Event(- event_type=response['object'],- params=kwargs,- result='Success',- returns={"content":- response['choices'][0]['message']['content']},- action_type='llm',- model=response['model'],- prompt=kwargs['messages'],+ self.client.record(LLMEvent(init_timestamp=init_timestamp,+ params=kwargs["invocation_params"],+ agent_id=check_call_stack_for_agent_id(),+ prompt=kwargs['messages'],+ completion={"content":+ response['choices'][0]['message']['content']},+ model=response['model'],prompt_tokens=response.get('usage',{}).get('prompt_tokens'),completion_tokens=response.get('usage',@@ -102,17 +95,15 @@ class LlmTracker:except:# v1.0.0+ responses are objectstry:- self.client.record(Event(- event_type=response.object,- params=kwargs,- result='Success',- returns={- # TODO: Will need to make the completion the key for content, splat out the model dump+ self.client.record(LLMEvent(+ init_timestamp=init_timestamp,+ params=kwargs["invocation_params"],+ agent_id=check_call_stack_for_agent_id(),+ # TODO: Will need to make the completion the key for content, splat out the model dump+ prompt=kwargs['messages'],+ completion={"content": response.choices[0].message.model_dump()},- action_type='llm',model=response.model,- prompt=kwargs['messages'],- init_timestamp=init_timestamp,prompt_tokens=response.usage.prompt_tokens,completion_tokens=response.usage.completion_tokens))@@ -140,18 +131,14 @@ class LlmTracker:role = choices[0].delta.roleif self.event_stream == None:- agent_id = check_call_stack_for_agent_id()- self.event_stream = Event(- event_type='openai chat completion stream',- agent_id=agent_id,+ self.event_stream = LLMEvent(+ init_timestamp=init_timestamp,params=kwargs,- result='Success',- returns={"finish_reason": None,- "content": token},- action_type='llm',- model=model,+ agent_id=check_call_stack_for_agent_id(),prompt=kwargs["messages"],- init_timestamp=init_timestamp+ completion={"finish_reason": None,+ "content": token},+ model=model)else:if token == None:@@ -199,19 +186,15 @@ class LlmTracker:# v1.0.0+ responses are objectstry:- agent_id = check_call_stack_for_agent_id()- self.client.record(Event(- event_type=response.object,- agent_id=agent_id,+ self.client.record(LLMEvent(+ init_timestamp=init_timestamp,params=kwargs,- result='Success',- returns={+ agent_id=check_call_stack_for_agent_id(),+ prompt=kwargs['messages'],+ completion={# TODO: Will need to make the completion the key for content, splat out the model dump"content": response.choices[0].message.model_dump()},- action_type='llm',model=response.model,- prompt=kwargs['messages'],- init_timestamp=init_timestamp,prompt_tokens=response.usage.prompt_tokens,completion_tokens=response.usage.completion_tokens))