Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
added callback handler
reibs7055571f263b
diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.pyindex efd2cacf..4106136d 100644--- a/agentops/langchain_callback_handler.py+++ b/agentops/langchain_callback_handler.py@@ -16,12 +16,11 @@ from typing import Any, Dict, List, Optional, Sequenceclass LangchainCallbackHandler(BaseCallbackHandler):- def __init__(self, api_key: str, tags: [str] = None):- self.ao_client = AOClient(api_key=api_key)- self.ao_client.start_session(tags)+ def __init__(self, api_key: str, tags: List[str] = None):+ self.ao_client = AOClient(api_key=api_key, tags=tags)# keypair <run_id: str, Event>- self.events = {}+ self.events: Dict[Any, Event] = {}# LLM Callbacksdef on_llm_start(@@ -35,9 +34,22 @@ class LangchainCallbackHandler(BaseCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> Any:+ # print(f"{serialized=}")+ # serialized={'lc': 1, 'type': 'constructor', 'id': ['langchain', 'chat_models', 'openai', 'ChatOpenAI'], 'kwargs': {'openai_api_key': {'lc': 1, 'type': 'secret', 'id': ['OPENAI_API_KEY']}}}++ # print(f"{prompts=}")+ # print(f"{run_id=}")+ # print(f"{parent_run_id=}")+ # print(f"{tags=}")+ # print(f"{metadata=}")+ # print(f"{kwargs=}")+ # kwargs={'invocation_params': {'model': 'gpt-3.5-turbo', 'model_name': 'gpt-3.5-turbo', 'request_timeout': None, 'max_tokens': None, 'stream': False, 'n': 1, 'temperature': 0.7, '_type': 'openai-chat', 'stop': ['Observation:']}, 'options': {'stop': ['Observation:']}, 'name': None}+self.events[run_id] = Event(- event_type="langchain_llm",+ event_type="llm",tags=tags,+ model=kwargs['invocation_params']['model'],+ params={**kwargs, **metadata},prompt="\n--\n".join(prompts),init_timestamp=get_ISO_time())@@ -64,7 +76,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):**kwargs: Any,) -> Any:self.events[run_id].end_timestamp = get_ISO_time()-+ self.events[run_id].returns = response.generations[0][0].message.contentif len(response.generations) > 0:self.events[run_id].result = "Success"else:@@ -82,7 +94,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):**kwargs: Any,) -> Any:self.events[run_id] = Event(- event_type="langchain_llm_chain",+ event_type="chain",init_timestamp=get_ISO_time())@@ -113,6 +125,24 @@ class LangchainCallbackHandler(BaseCallbackHandler):self.ao_client.record(self.events[run_id])# Tool callbacks++ def on_tool_start(+ self,+ serialized: Dict[str, Any],+ input_str: str,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ metadata: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> Any:+ """Run when tool starts running."""+ self.events[run_id] = Event(+ event_type="tool",+ init_timestamp=get_ISO_time()+ )+def on_tool_end(self,output: str,@@ -121,11 +151,10 @@ class LangchainCallbackHandler(BaseCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- self.ao_client.record(Event(- event_type="langchain_tool_usage",- result="Success",- init_timestamp=get_ISO_time()- ))+ self.events[run_id].end_timestamp = get_ISO_time()+ self.events[run_id].result = "Success"++ self.ao_client.record(self.events[run_id])def on_tool_error(self,@@ -136,11 +165,16 @@ class LangchainCallbackHandler(BaseCallbackHandler):**kwargs: Any,) -> Any:self.ao_client.record(Event(- event_type="langchain_tool_usage",+ event_type="",result="Fail",init_timestamp=get_ISO_time()))+ self.events[run_id].end_timestamp = get_ISO_time()+ self.events[run_id].result = "Fail"++ self.ao_client.record(self.events[run_id])+# Retriever callbacksasync def on_retriever_start(self,@@ -154,7 +188,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):**kwargs: Any,) -> None:self.events[run_id] = Event(- event_type="langchain_llm_retriever",+ event_type="retriever",init_timestamp=get_ISO_time())@@ -168,7 +202,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):**kwargs: Any,) -> None:self.events[run_id].end_timestamp = get_ISO_time()- self.events[run_id].result = "Successful"+ self.events[run_id].result = "Success"self.ao_client.record(self.events[run_id])