Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

async callback handler

reibs564f6ebad475
diff --git a/agentops/__init__.py b/agentops/__init__.py
index 4dc51d37..523208eb 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -2,3 +2,4 @@ from .client import Client
from .event import Event
from .logger import AgentOpsLogger
from .helpers import Models
+from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandler
diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.py
index 08981edb..54954692 100644
--- a/agentops/langchain_callback_handler.py
+++ b/agentops/langchain_callback_handler.py
@@ -9,7 +9,7 @@ from agentops import Client as AOClient
from agentops import Event
from tenacity import RetryCallState
-from langchain.callbacks.base import BaseCallbackHandler
+from langchain.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler
from agentops.helpers import get_ISO_time
from typing import Any, Dict, List, Optional, Sequence
@@ -122,6 +122,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):
) -> Any:
self.events[run_id].end_timestamp = get_ISO_time()
self.events[run_id].result = "Fail"
+ self.events[run_id].returns = str(error)
self.ao_client.record(self.events[run_id])
@@ -137,6 +138,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Any:
+ print('sync tool use')
"""Run when tool starts running."""
self.events[run_id] = Event(
event_type="tool",
@@ -281,3 +283,277 @@ class LangchainCallbackHandler(BaseCallbackHandler):
@property
def session_id(self):
return self.ao_client.session.session_id
+
+
+class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
+ """Callback handler for Langchain agents."""
+
+ def __init__(self, api_key: str, tags: List[str] = None):
+ self.ao_client = AOClient(api_key=api_key, tags=tags, override=False)
+
+ # keypair <run_id: str, Event>
+ self.events: Dict[Any, Event] = {}
+
+ # LLM Callbacks
+ async def on_llm_start(
+ self,
+ serialized: Dict[str, Any],
+ prompts: List[str],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events[run_id] = Event(
+ event_type="llm",
+ action_type='llm',
+ tags=tags,
+ model=kwargs['invocation_params']['model'],
+ params={**kwargs, **metadata},
+ prompt=prompts[0],
+ init_timestamp=get_ISO_time()
+ )
+
+ async def on_llm_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].result = "Fail"
+
+ self.ao_client.record(self.events[run_id])
+
+ async def on_llm_end(
+ self,
+ response: LLMResult,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].returns = response.generations[0][0].message.content
+ self.events[run_id].prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
+ self.events[run_id].completion_tokens = response.llm_output['token_usage']['completion_tokens']
+
+ if len(response.generations) > 0:
+ self.events[run_id].result = "Success"
+ else:
+ self.events[run_id].result = "Fail"
+
+ self.ao_client.record(self.events[run_id])
+
+ # Chain callbacks
+ async def on_chain_start(
+ self,
+ serialized: Dict[str, Any],
+ inputs: Dict[str, Any],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ print('on_chain_start')
+ self.events[run_id] = Event(
+ event_type="chain",
+ init_timestamp=get_ISO_time(),
+ tags=tags,
+ params={**inputs, **kwargs, **metadata},
+ )
+
+ async def on_chain_end(
+ self,
+ outputs: Dict[str, Any],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ print('on_chain_end')
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].result = "Success"
+ self.events[run_id].returns = outputs
+
+ self.ao_client.record(self.events[run_id])
+
+ async def on_chain_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ print('on_chain_error')
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].result = "Fail"
+ self.events[run_id].returns = str(error)
+
+ self.ao_client.record(self.events[run_id])
+
+ # Tool callbacks
+ async def on_tool_start(
+ self,
+ serialized: Dict[str, Any],
+ input_str: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ """Run when tool starts running."""
+ print('on_tool_start')
+ self.events[run_id] = Event(
+ event_type="tool",
+ init_timestamp=get_ISO_time(),
+ tags=tags,
+ params={**serialized, **metadata},
+ )
+
+ async def on_tool_end(
+ self,
+ output: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ print('on_tool_end')
+ # Tools are capable of failing `on_tool_end` quietly.
+ # This is a workaround to make sure we can log it as an error.
+ if kwargs.get('name') == '_Exception':
+ self.events[run_id].result = "Fail"
+ else:
+ self.events[run_id].result = "Success"
+
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].returns = output
+
+ self.ao_client.record(self.events[run_id])
+
+ async def on_tool_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].result = "Fail"
+ self.events[run_id].returns = str(error)
+
+ self.ao_client.record(self.events[run_id])
+
+ # Retriever callbacks
+ async def on_retriever_start(
+ self,
+ serialized: Dict[str, Any],
+ query: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> None:
+ self.events[run_id] = Event(
+ event_type="retriever",
+ init_timestamp=get_ISO_time()
+ )
+
+ async def on_retriever_end(
+ self,
+ documents: Sequence[Document],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].result = "Success"
+
+ self.ao_client.record(self.events[run_id])
+
+ async def on_retriever_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].result = "Fail"
+
+ self.ao_client.record(self.events[run_id])
+
+ # Agent callbacks
+ async def on_agent_action(
+ self,
+ action: AgentAction,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ """Run on agent action."""
+ self.events[run_id] = Event(
+ event_type="agent",
+ init_timestamp=get_ISO_time(),
+ params={**kwargs},
+ )
+
+ async def on_agent_finish(
+ self,
+ finish: AgentFinish,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ """Run on agent finish."""
+ self.events[run_id].end_timestamp = get_ISO_time()
+ self.events[run_id].result = "Success"
+ self.events[run_id].returns = finish
+
+ self.ao_client.record(self.events[run_id])
+
+ # TODO: Create a way for the end user to set this based on their conditions
+ self.ao_client.end_session("Success")
+
+ # Misc.
+ async def on_retry(
+ self,
+ retry_state: RetryCallState,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ """Run on a retry event."""
+ event = Event(
+ event_type="retry",
+ init_timestamp=get_ISO_time(),
+ end_timestamp=get_ISO_time(),
+ params={**kwargs},
+ result="Indeterminate",
+ returns=retry_state
+ )
+ self.ao_client.record(event)
+
+ @property
+ def session_id(self):
+ return self.ao_client.session.session_id