Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Fixed bug where multiple events needed to be stored in same run_id
Howard Gil086ca3167e60
diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.pyindex b8539fcd..dc4bbb9b 100644--- a/agentops/langchain_callback_handler.py+++ b/agentops/langchain_callback_handler.py@@ -36,7 +36,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):if v is not None}, override=False)# keypair <run_id: str, ActionEvent>- self.events: Dict[Any, Event] = {}+ self.events: Dict[str, Event] = {}# LLM Callbacksdef on_llm_start(@@ -50,7 +50,17 @@ class LangchainCallbackHandler(BaseCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> Any:- self.events[run_id] = LLMEvent(+ key = "llm_" + str(run_id)++ import inspect+ print(inspect.currentframe().f_code.co_name)+ print("run_id: ", run_id)+ print("parent_run_id: ", parent_run_id)++ if run_id in self.events:+ print("already an event at this run_id")++ self.events[key] = LLMEvent(# tags=tags, # TODO: Are these tags coming from langchain?params={**serialized, **kwargs, **({} if metadata is None else metadata)},model=kwargs['invocation_params']['model'],@@ -65,7 +75,8 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- llmEvent: LLMEvent = self.events[run_id]+ key = "llm_" + str(run_id)+ llmEvent: LLMEvent = self.events[key]errorEvent = ErrorEvent(trigger_event=llmEvent,details=str(error), # TODO: do we need to str(error)@@ -82,7 +93,8 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- llmEvent: LLMEvent = self.events[run_id]+ key = "llm_" + str(run_id)+ llmEvent: LLMEvent = self.events[key]llmEvent.end_timestamp = get_ISO_time()llmEvent.returns = {@@ -101,7 +113,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):if len(response.generations) == 0:errorEvent = ErrorEvent(- trigger_event=self.events[run_id],+ trigger_event=self.events[key],details="on_llm_end: No generations", # TODO: more descriptive errortimestamp=get_ISO_time())@@ -119,7 +131,18 @@ class LangchainCallbackHandler(BaseCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> Any:- self.events[run_id] = ActionEvent(+ key = "chain_" + str(run_id)++ import inspect++ print(inspect.currentframe().f_code.co_name)+ print("run_id: ", run_id)+ print("parent_run_id: ", parent_run_id)++ if run_id in self.events:+ print("already an event at this run_id")++ self.events[key] = ActionEvent(params={**serialized, **inputs, **kwargs, **({} if metadata is None else metadata)},action_type="chain"@@ -134,7 +157,9 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- actionEvent: ActionEvent = self.events[run_id]+ key = "chain_" + str(run_id)++ actionEvent: ActionEvent = self.events[key]actionEvent.returns = outputsactionEvent.end_timestamp = get_ISO_time()@@ -149,7 +174,8 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- actionEvent: ActionEvent = self.events[run_id]+ key = "chain_" + str(run_id)+ actionEvent: ActionEvent = self.events[key]self.ao_client.record(actionEvent)# TODO: do we need to str(error)@@ -168,7 +194,17 @@ class LangchainCallbackHandler(BaseCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> Any:- self.events[run_id] = ToolEvent(+ key = "tool_" + str(run_id)++ import inspect+ print(inspect.currentframe().f_code.co_name)+ print("run_id: ", run_id)+ print("parent_run_id: ", parent_run_id)++ if run_id in self.events:+ print("already an event at this run_id")++ self.events[key] = ToolEvent(# tags=tags,# TODO: we use serialized here but kwargs in on_llm_startparams={**serialized, **({} if metadata is None else metadata)},@@ -184,7 +220,8 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- toolEvent: ToolEvent = self.events[run_id]+ key = "tool_" + str(run_id)+ toolEvent: ToolEvent = self.events[key]toolEvent.end_timestamp = get_ISO_time()toolEvent.returns = output@@ -204,7 +241,9 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- toolEvent: ToolEvent = self.events[run_id]+ key = "tool_" + str(run_id)++ toolEvent: ToolEvent = self.events[key]self.ao_client.record(toolEvent)# TODO: do we need to str(error)@@ -223,7 +262,17 @@ class LangchainCallbackHandler(BaseCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> None:- self.events[run_id] = ActionEvent(+ key = "retreiver_" + str(run_id)++ import inspect+ print(inspect.currentframe().f_code.co_name)+ print("run_id: ", run_id)+ print("parent_run_id: ", parent_run_id)++ if run_id in self.events:+ print("already an event at this run_id")++ self.events[key] = ActionEvent(params={**serialized, **kwargs, "query": query, **({} if metadata is None else metadata)},action_type="retreiver"# tags=tags,@@ -238,7 +287,9 @@ class LangchainCallbackHandler(BaseCallbackHandler):tags: Optional[List[str]] = None,**kwargs: Any,) -> None:- actionEvent: ActionEvent = self.events[run_id]+ key = "retreiver_" + str(run_id)++ actionEvent: ActionEvent = self.events[key]actionEvent.logs = documents # TODO: Adding this. Might want to add elsewhere e.g. paramsactionEvent.end_timestamp = get_ISO_time()@@ -254,7 +305,9 @@ class LangchainCallbackHandler(BaseCallbackHandler):tags: Optional[List[str]] = None,**kwargs: Any,) -> None:- actionEvent: ActionEvent = self.events[run_id]+ key = "retreiver_" + str(run_id)++ actionEvent: ActionEvent = self.events[key]self.ao_client.record(actionEvent)# TODO: do we need to str(error)@@ -270,7 +323,14 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- self.events[run_id] = ActionEvent(+ key = "agent_" + str(run_id)++ import inspect+ print(inspect.currentframe().f_code.co_name)+ print("run_id: ", run_id)+ print("parent_run_id: ", parent_run_id)++ self.events[key] = ActionEvent(params={"action": action, **kwargs},action_type="agent")@@ -283,7 +343,9 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- actionEvent: ActionEvent = self.events[run_id]+ key = "agent_" + str(run_id)++ actionEvent: ActionEvent = self.events[key]actionEvent.returns = finish.to_json()actionEvent.end_timestamp = get_ISO_time()@@ -301,6 +363,13 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:+ key = "retry_" + str(run_id)++ import inspect+ print(inspect.currentframe().f_code.co_name)+ print("run_id: ", run_id)+ print("parent_run_id: ", parent_run_id)+actionEvent = ActionEvent(params={**kwargs},returns=retry_state,@@ -349,7 +418,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> Any:- self.events[run_id] = LLMEvent(+ self.events[key] = LLMEvent(event_type="llm",action_type='llm',tags=tags,@@ -367,10 +436,10 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].result = "Fail"+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].result = "Fail"- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])async def on_llm_end(self,@@ -380,21 +449,21 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].returns = {+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].returns = {"content": response.generations[0][0].message.content,"generations": response.generations}if response.llm_output is not None:- self.events[run_id].prompt_tokens = response.llm_output['token_usage']['prompt_tokens']- self.events[run_id].completion_tokens = response.llm_output['token_usage']['completion_tokens']+ self.events[key].prompt_tokens = response.llm_output['token_usage']['prompt_tokens']+ self.events[key].completion_tokens = response.llm_output['token_usage']['completion_tokens']if len(response.generations) > 0:- self.events[run_id].result = "Success"+ self.events[key].result = "Success"else:- self.events[run_id].result = "Fail"+ self.events[key].result = "Fail"- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])# Chain callbacksasync def on_chain_start(@@ -408,7 +477,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> Any:- self.events[run_id] = ActionEvent(+ self.events[key] = ActionEvent(event_type="chain",init_timestamp=get_ISO_time(),tags=tags,@@ -425,11 +494,11 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].result = "Success"- self.events[run_id].returns = outputs+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].result = "Success"+ self.events[key].returns = outputs- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])async def on_chain_error(self,@@ -439,11 +508,11 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].result = "Fail"- self.events[run_id].returns = str(error)+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].result = "Fail"+ self.events[key].returns = str(error)- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])# Tool callbacksasync def on_tool_start(@@ -458,7 +527,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):**kwargs: Any,) -> Any:"""Run when tool starts running."""- self.events[run_id] = ToolEvent(+ self.events[key] = ToolEvent(event_type="tool",init_timestamp=get_ISO_time(),tags=tags,@@ -476,14 +545,14 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):# Tools are capable of failing `on_tool_end` quietly.# This is a workaround to make sure we can log it as an error.if kwargs.get('name') == '_Exception':- self.events[run_id].result = "Fail"+ self.events[key].result = "Fail"else:- self.events[run_id].result = "Success"+ self.events[key].result = "Success"- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].returns = output+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].returns = output- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])async def on_tool_error(self,@@ -493,11 +562,11 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].result = "Fail"- self.events[run_id].returns = str(error)+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].result = "Fail"+ self.events[key].returns = str(error)- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])# Retriever callbacksasync def on_retriever_start(@@ -511,7 +580,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> None:- self.events[run_id] = ActionEvent(+ self.events[key] = ActionEvent(event_type="retriever",init_timestamp=get_ISO_time())@@ -525,10 +594,10 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):tags: Optional[List[str]] = None,**kwargs: Any,) -> None:- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].result = "Success"+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].result = "Success"- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])async def on_retriever_error(self,@@ -539,10 +608,10 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):tags: Optional[List[str]] = None,**kwargs: Any,) -> None:- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].result = "Fail"+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].result = "Fail"- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])# Agent callbacksasync def on_agent_action(@@ -554,7 +623,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):**kwargs: Any,) -> Any:"""Run on agent action."""- self.events[run_id] = ActionEvent(+ self.events[key] = ActionEvent(event_type="agent",init_timestamp=get_ISO_time(),params={**kwargs},@@ -569,11 +638,11 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):**kwargs: Any,) -> Any:"""Run on agent finish."""- self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].result = "Success"- self.events[run_id].returns = finish.to_json()+ self.events[key].end_timestamp = get_ISO_time()+ self.events[key].result = "Success"+ self.events[key].returns = finish.to_json()- self.ao_client.record(self.events[run_id])+ self.ao_client.record(self.events[key])# TODO: Create a way for the end user to set this based on their conditionsself.ao_client.end_session("Success")