Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
rename prompt and completion (#131)
Shawn Qiu4018ac78dbc0
diff --git a/agentops/event.py b/agentops/event.pyindex b4b10d0e..8e1ab163 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -76,9 +76,9 @@ class LLMEvent(Event):agent_id(UUID, optional): The unique identifier of the agent that triggered the event.thread_id(UUID, optional): The unique identifier of the contextual thread that a message pertains to.- prompt_message(str, list, optional): The message or messages that were used to prompt the LLM. Preferably in ChatML format which is more fully supported by AgentOps.+ prompt(str, list, optional): The message or messages that were used to prompt the LLM. Preferably in ChatML format which is more fully supported by AgentOps.prompt_tokens(int, optional): The number of tokens in the prompt message.- completion_message(str, object, optional): The message or returned by the LLM. Preferably in ChatML format which is more fully supported by AgentOps.+ completion(str, object, optional): The message or returned by the LLM. Preferably in ChatML format which is more fully supported by AgentOps.completion_tokens(int, optional): The number of tokens in the completion message.model(Models, str, optional): LLM model e.g. "gpt-4". Models defined in enums.Models are more fully supported by AgentOps e.g. extra features in dashboard.@@ -87,9 +87,9 @@ class LLMEvent(Event):event_type: str = EventType.LLM.valueagent_id: Optional[UUID] = Nonethread_id: Optional[UUID] = None- prompt_message: str | List = None+ prompt: str | List = Noneprompt_tokens: Optional[int] = None- completion_message: str | object = None+ completion: str | object = Nonecompletion_tokens: Optional[int] = Nonemodel: Optional[Models | str] = Nonediff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.pyindex daca246d..a79d3ea9 100644--- a/agentops/langchain_callback_handler.py+++ b/agentops/langchain_callback_handler.py@@ -64,7 +64,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):**({} if metadata is None else metadata),**kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logsmodel=kwargs['invocation_params']['model'],- prompt_message=prompts[0]+ prompt=prompts[0]# tags=tags # TODO)@@ -99,7 +99,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):}llm_event.end_timestamp = get_ISO_time()if response.llm_output is not None:- llm_event.completion_message = response.generations[0][0].message.content # TODO+ llm_event.completion = response.generations[0][0].message.content # TODOllm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']self.ao_client.record(llm_event)@@ -363,7 +363,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):**({} if metadata is None else metadata),**kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logsmodel=kwargs['invocation_params']['model'],- prompt_message=prompts[0]+ prompt=prompts[0])@debug_print_function_params@@ -424,7 +424,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):}llm_event.end_timestamp = get_ISO_time()if response.llm_output is not None:- llm_event.completion_message = response.generations[0][0].message.content # TODO+ llm_event.completion = response.generations[0][0].message.content # TODOllm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']self.ao_client.record(llm_event)diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 6b03c840..efb32a3c 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -50,8 +50,8 @@ class LlmTracker:if finish_reason:self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt_message = kwargs["messages"]- self.llm_event.completion_message = {"role": "assistant", "content": self.completion}+ self.llm_event.prompt = kwargs["messages"]+ self.llm_event.completion = {"role": "assistant", "content": self.completion}self.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion}self.llm_event.model = modelself.llm_event.end_timestamp = get_ISO_time()@@ -87,9 +87,9 @@ class LlmTracker:# v0.0.0 responses are dictstry:self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt_message = kwargs["messages"]+ self.llm_event.prompt = kwargs["messages"]self.llm_event.prompt_tokens = response['usage']['prompt_tokens']- self.llm_event.completion_message = response['choices'][0]['message']['content']+ self.llm_event.completion = response['choices'][0]['message']['content']self.llm_event.completion_tokens = response['usage']['completion_tokens']self.llm_event.returns = {"content": response['choices'][0]['message']['content']}self.llm_event.model = response["model"]@@ -134,8 +134,8 @@ class LlmTracker:if finish_reason:self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt_message = kwargs["messages"]- self.llm_event.completion_message = {"role": "assistant", "content": self.completion}+ self.llm_event.prompt = kwargs["messages"]+ self.llm_event.completion = {"role": "assistant", "content": self.completion}self.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion,"function_call": function_call, "tool_calls": tool_calls, "role": role}self.llm_event.model = model@@ -179,9 +179,9 @@ class LlmTracker:# v1.0.0+ responses are objectstry:self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt_message = kwargs["messages"]+ self.llm_event.prompt = kwargs["messages"]self.llm_event.prompt_tokens = response.usage.prompt_tokens- self.llm_event.completion_message = response.choices[0].message.model_dump().get('content')+ self.llm_event.completion = response.choices[0].message.model_dump().get('content')self.llm_event.completion_tokens = response.usage.completion_tokensself.llm_event.returns = response.model_dump()self.llm_event.model = response.model