Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Autogen support 2 (#220)

Braelyn Boynton4f98d2ad9d8c
diff --git a/agentops/__init__.py b/agentops/__init__.py
index 46ae55a9..cda6079c 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -10,7 +10,7 @@ from .decorators import record_function
from .agent import track_agent
from .log_config import logger
try:
- from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandler
+ from .partners.langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandler
except ModuleNotFoundError:
pass
diff --git a/agentops/client.py b/agentops/client.py
index f6e54d79..67c0dbfa 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -119,7 +119,8 @@ class Client(metaclass=MetaClient):
if framework == 'autogen':
try:
import autogen
- autogen.runtime_logging.start(logger_type="agentops")
+ from .partners.autogen_logger import AutogenLogger
+ autogen.runtime_logging.start(logger=AutogenLogger())
except ImportError:
pass
except Exception as e:
diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.py
index 68383a36..fb2e9c83 100644
--- a/agentops/langchain_callback_handler.py
+++ b/agentops/langchain_callback_handler.py
@@ -13,6 +13,7 @@ from tenacity import RetryCallState
from agentops import Client as AOClient
from agentops import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
from agentops.helpers import get_ISO_time
+import logging
from .helpers import debug_print_function_params
@@ -49,6 +50,9 @@ class LangchainCallbackHandler(BaseCallbackHandler):
max_queue_size: Optional[int] = None,
tags: Optional[List[str]] = None):
+ logging.warning('🚨Importing the Langchain Callback Handler from here is deprecated. Please import with '
+ '`from agentops.partners import LangchainCallbackHandler`')
+
client_params: Dict[str, Any] = {
'api_key': api_key,
'endpoint': endpoint,
diff --git a/agentops/partners/autogen_logger.py b/agentops/partners/autogen_logger.py
new file mode 100644
index 00000000..03e9e0fd
--- /dev/null
+++ b/agentops/partners/autogen_logger.py
@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import logging
+import threading
+import uuid
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union, TypeVar, Callable
+
+import agentops
+from openai import AzureOpenAI, OpenAI
+from openai.types.chat import ChatCompletion
+
+from autogen.logger.base_logger import BaseLogger, LLMConfig
+from autogen.logger.logger_utils import get_current_ts, to_dict
+
+from agentops.enums import EndState
+
+from agentops import LLMEvent, ToolEvent, ActionEvent
+from uuid import uuid4
+
+if TYPE_CHECKING:
+ from autogen import Agent, ConversableAgent, OpenAIWrapper
+
+logger = logging.getLogger(__name__)
+lock = threading.Lock()
+
+__all__ = ("AutogenLogger",)
+
+F = TypeVar("F", bound=Callable[..., Any])
+
+
+class AutogenLogger(BaseLogger):
+ agent_store: [{"agentops_id": str, "autogen_id": str}] = []
+
+ def start(self) -> str:
+ pass
+
+ def _get_agentops_id_from_agent(self, autogen_id: str) -> str:
+ for agent in self.agent_store:
+ if agent["autogen_id"] == autogen_id:
+ return agent["agentops_id"]
+
+ def log_chat_completion(
+ self,
+ invocation_id: uuid.UUID,
+ client_id: int,
+ wrapper_id: int,
+ agent: Union[str, Agent],
+ request: Dict[str, Union[float, str, List[Dict[str, str]]]],
+ response: Union[str, ChatCompletion],
+ is_cached: int,
+ cost: float,
+ start_time: str,
+ ) -> None:
+ """ Records an LLMEvent to AgentOps session """
+ end_time = get_current_ts()
+
+ completion = response.choices[len(response.choices)-1]
+
+ llm_event = LLMEvent(prompt=request['messages'], completion=completion.message, model=response.model)
+ llm_event.init_timestamp = start_time
+ llm_event.end_timestamp = end_time
+ llm_event.agent_id = self._get_agentops_id_from_agent(str(id(agent)))
+ agentops.record(llm_event)
+
+ def log_new_agent(self, agent: ConversableAgent, init_args: Dict[str, Any]) -> None:
+ """ Calls agentops.create_agent """
+ ao_agent_id = agentops.create_agent(agent.name, str(uuid4()))
+ self.agent_store.append({'agentops_id': ao_agent_id, 'autogen_id': str(id(agent))})
+
+ def log_event(self, source: Union[str, Agent], name: str, **kwargs: Dict[str, Any]) -> None:
+ """ Records an ActionEvent to AgentOps session """
+ event = ActionEvent(action_type=name)
+ agentops_id = self._get_agentops_id_from_agent(str(id(source)))
+ event.agent_id = agentops_id
+ event.params = kwargs
+ agentops.record(event)
+
+ def log_function_use(
+ self, source: Union[str, Agent], function: F, args: Dict[str, Any], returns: any
+ ):
+ """ Records a ToolEvent to AgentOps session """
+ event = ToolEvent()
+ agentops_id = self._get_agentops_id_from_agent(str(id(source)))
+ event.agent_id = agentops_id
+ event.function = function # TODO: this is not a parameter
+ event.params = args
+ event.returns = returns
+ event.name = getattr(function, '_name')
+ agentops.record(event)
+
+ def log_new_wrapper(self, wrapper: OpenAIWrapper, init_args: Dict[str, Union[LLMConfig, List[LLMConfig]]]) -> None:
+ pass
+
+ def log_new_client(
+ self, client: Union[AzureOpenAI, OpenAI], wrapper: OpenAIWrapper, init_args: Dict[str, Any]
+ ) -> None:
+ pass
+
+ def stop(self) -> None:
+ """ Ends AgentOps session """
+ agentops.end_session(end_state=EndState.INDETERMINATE.value)
+
+ def get_connection(self) -> None:
+ """ Method intentionally left blank """
+ pass
diff --git a/agentops/partners/langchain_callback_handler.py b/agentops/partners/langchain_callback_handler.py
new file mode 100644
index 00000000..e2abf3cf
--- /dev/null
+++ b/agentops/partners/langchain_callback_handler.py
@@ -0,0 +1,668 @@
+from typing import Dict, Any, List, Optional, Sequence, Union
+from collections import defaultdict
+from uuid import UUID
+
+from langchain_core.agents import AgentFinish, AgentAction
+from langchain_core.documents import Document
+from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult
+from langchain.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler
+from langchain_core.messages import BaseMessage
+
+from tenacity import RetryCallState
+
+from agentops import Client as AOClient
+from agentops import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
+from agentops.helpers import get_ISO_time
+
+from ..helpers import debug_print_function_params
+
+
+def get_model_from_kwargs(kwargs: any) -> str:
+ if 'model' in kwargs['invocation_params']:
+ return kwargs['invocation_params']['model']
+ elif '_type' in kwargs['invocation_params']:
+ return kwargs['invocation_params']['_type']
+ else:
+ return 'unknown_model'
+
+
+class Events:
+ llm: Dict[str, LLMEvent] = {}
+ tool: Dict[str, ToolEvent] = {}
+ chain: Dict[str, ActionEvent] = {}
+ retriever: Dict[str, ActionEvent] = {}
+ error: Dict[str, ErrorEvent] = {}
+
+
+class LangchainCallbackHandler(BaseCallbackHandler):
+ """Callback handler for Langchain agents."""
+
+ def __init__(self, api_key: Optional[str] = None,
+ endpoint: Optional[str] = None,
+ max_wait_time: Optional[int] = None,
+ max_queue_size: Optional[int] = None,
+ tags: Optional[List[str]] = None):
+
+ client_params: Dict[str, Any] = {
+ 'api_key': api_key,
+ 'endpoint': endpoint,
+ 'max_wait_time': max_wait_time,
+ 'max_queue_size': max_queue_size,
+ 'tags': tags
+ }
+
+ self.ao_client = AOClient(**{k: v for k, v in client_params.items()
+ if v is not None}, override=False)
+ self.agent_actions: Dict[UUID, List[ActionEvent]] = defaultdict(list)
+ self.events = Events()
+
+ @debug_print_function_params
+ def on_llm_start(
+ self,
+ serialized: Dict[str, Any],
+ prompts: List[str],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events.llm[str(run_id)] = LLMEvent(
+ params={**serialized,
+ **({} if metadata is None else metadata),
+ **kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logs
+ model=get_model_from_kwargs(kwargs),
+ prompt=prompts[0]
+ # tags=tags # TODO
+ )
+
+ @debug_print_function_params
+ def on_llm_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ llm_event: LLMEvent = self.events.llm[str(run_id)]
+ self.ao_client.record(llm_event)
+
+ error_event = ErrorEvent(trigger_event=llm_event, exception=error)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ def on_llm_end(
+ self,
+ response: LLMResult,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ llm_event: LLMEvent = self.events.llm[str(run_id)]
+ llm_event.returns = {
+ "content": response.generations[0][0].text,
+ "generations": response.generations
+ }
+ llm_event.end_timestamp = get_ISO_time()
+ llm_event.completion = response.generations[0][0].text
+ if response.llm_output is not None:
+ llm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
+ llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']
+
+ self.ao_client.record(llm_event)
+
+ if len(response.generations) == 0:
+ # TODO: more descriptive error
+ error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations")
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ def on_chain_start(
+ self,
+ serialized: Dict[str, Any],
+ inputs: Dict[str, Any],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events.chain[str(run_id)] = ActionEvent(
+ params={**serialized,
+ **inputs,
+ **({} if metadata is None else metadata),
+ **kwargs},
+ action_type="chain"
+ )
+
+ @debug_print_function_params
+ def on_chain_end(
+ self,
+ outputs: Dict[str, Any],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ action_event: ActionEvent = self.events.chain[str(run_id)]
+ action_event.returns = outputs
+ action_event.end_timestamp = get_ISO_time()
+ self.ao_client.record(action_event)
+
+ @debug_print_function_params
+ def on_chain_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ action_event: ActionEvent = self.events.chain[str(run_id)]
+ self.ao_client.record(action_event)
+
+ error_event = ErrorEvent(trigger_event=action_event, exception=error)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ def on_tool_start(
+ self,
+ serialized: Dict[str, Any],
+ input_str: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ inputs: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events.tool[str(run_id)] = ToolEvent(
+ params=input_str if inputs is None else inputs,
+ name=serialized['name'],
+ logs={**serialized,
+ 'tags': tags,
+ **({} if metadata is None else metadata),
+ **({} if inputs is None else inputs),
+ **kwargs}
+ )
+
+ @debug_print_function_params
+ def on_tool_end(
+ self,
+ output: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ tool_event: ToolEvent = self.events.tool[str(run_id)]
+ tool_event.end_timestamp = get_ISO_time()
+ tool_event.returns = output
+ self.ao_client.record(tool_event)
+
+ # Tools are capable of failing `on_tool_end` quietly.
+ # This is a workaround to make sure we can log it as an error.
+ if kwargs.get('name') == '_Exception':
+ error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ def on_tool_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ tool_event: ToolEvent = self.events.tool[str(run_id)]
+ self.ao_client.record(tool_event)
+
+ error_event = ErrorEvent(trigger_event=tool_event, exception=error)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ def on_retriever_start(
+ self,
+ serialized: Dict[str, Any],
+ query: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> None:
+ self.events.retriever[str(run_id)] = ActionEvent(
+ params={**serialized,
+ "query": query,
+ **({} if metadata is None else metadata),
+ ** kwargs},
+ action_type="retriever"
+ )
+
+ @debug_print_function_params
+ def on_retriever_end(
+ self,
+ documents: Sequence[Document],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ action_event: ActionEvent = self.events.retriever[str(run_id)]
+ action_event.logs = documents # TODO: Adding this. Might want to add elsewhere e.g. params
+ action_event.end_timestamp = get_ISO_time()
+ self.ao_client.record(action_event)
+
+ @debug_print_function_params
+ def on_retriever_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ action_event: ActionEvent = self.events.retriever[str(run_id)]
+ self.ao_client.record(action_event)
+
+ error_event = ErrorEvent(trigger_event=action_event, exception=error)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ def on_agent_action(
+ self,
+ action: AgentAction,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.agent_actions[run_id].append(ActionEvent(
+ params={"action": action,
+ **kwargs},
+ action_type="agent"
+ ))
+
+ @debug_print_function_params
+ def on_agent_finish(
+ self,
+ finish: AgentFinish,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ # Need to attach finish to some on_agent_action so just choosing the last one
+ self.agent_actions[run_id][-1].returns = finish.to_json()
+
+ for agentAction in self.agent_actions[run_id]:
+ self.ao_client.record(agentAction)
+
+ # TODO: Create a way for the end user to set this based on their conditions
+ # self.ao_client.end_session("Success") #TODO: calling end_session here causes "No current session"
+
+ @debug_print_function_params
+ def on_retry(
+ self,
+ retry_state: RetryCallState,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ action_event = ActionEvent(
+ params={**kwargs},
+ returns=str(retry_state),
+ action_type="retry",
+ # result="Indeterminate" # TODO: currently have no way of recording Indeterminate
+ )
+ self.ao_client.record(action_event)
+
+ @property
+ def session_id(self):
+ return self.ao_client.current_session_id
+
+
+class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
+ """Callback handler for Langchain agents."""
+
+ def __init__(self, api_key: Optional[str] = None,
+ endpoint: Optional[str] = None,
+ max_wait_time: Optional[int] = None,
+ max_queue_size: Optional[int] = None,
+ tags: Optional[List[str]] = None):
+
+ client_params: Dict[str, Any] = {
+ 'api_key': api_key,
+ 'endpoint': endpoint,
+ 'max_wait_time': max_wait_time,
+ 'max_queue_size': max_queue_size,
+ 'tags': tags
+ }
+
+ self.ao_client = AOClient(**{k: v for k, v in client_params.items()
+ if v is not None}, override=False)
+
+ self.events = Events()
+ self.agent_actions: Dict[UUID, List[ActionEvent]] = defaultdict(list)
+
+ @debug_print_function_params
+ async def on_llm_start(
+ self,
+ serialized: Dict[str, Any],
+ prompts: List[str],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events.llm[str(run_id)] = LLMEvent(
+ params={**serialized,
+ **({} if metadata is None else metadata),
+ **kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logs
+ model=kwargs['invocation_params']['model'],
+ prompt=prompts[0]
+ )
+
+ @debug_print_function_params
+ async def on_chat_model_start(
+ self,
+ serialized: Dict[str, Any],
+ messages: List[List[BaseMessage]],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ pass
+
+ @debug_print_function_params
+ async def on_llm_new_token(
+ self,
+ token: str,
+ *,
+ chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ pass
+
+ @debug_print_function_params
+ async def on_llm_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ llm_event: LLMEvent = self.events.llm[str(run_id)]
+ self.ao_client.record(llm_event)
+
+ error_event = ErrorEvent(trigger_event=llm_event, exception=error)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ async def on_llm_end(
+ self,
+ response: LLMResult,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ llm_event: LLMEvent = self.events.llm[str(run_id)]
+ llm_event.returns = {
+ "content": response.generations[0][0].text,
+ "generations": response.generations
+ }
+ llm_event.end_timestamp = get_ISO_time()
+ llm_event.completion = response.generations[0][0].text
+ if response.llm_output is not None:
+ llm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
+ llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']
+ self.ao_client.record(llm_event)
+
+ if len(response.generations) == 0:
+ # TODO: more descriptive error
+ error_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)], error_type="NoGenerations", details="on_llm_end: No generations")
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ async def on_chain_start(
+ self,
+ serialized: Dict[str, Any],
+ inputs: Dict[str, Any],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events.chain[str(run_id)] = ActionEvent(
+ params={**serialized,
+ **inputs,
+ **({} if metadata is None else metadata),
+ **kwargs},
+ action_type="chain"
+ )
+
+ @debug_print_function_params
+ async def on_chain_end(
+ self,
+ outputs: Dict[str, Any],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ action_event: ActionEvent = self.events.chain[str(run_id)]
+ action_event.returns = outputs
+ action_event.end_timestamp = get_ISO_time()
+ self.ao_client.record(action_event)
+
+ @debug_print_function_params
+ async def on_chain_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ action_event: ActionEvent = self.events.chain[str(run_id)]
+ self.ao_client.record(action_event)
+
+ error_event = ErrorEvent(trigger_event=action_event, exception=error)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ async def on_tool_start(
+ self,
+ serialized: Dict[str, Any],
+ input_str: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ inputs: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.events.tool[str(run_id)] = ToolEvent(
+ params=input_str if inputs is None else inputs,
+ name=serialized['name'],
+ logs={**serialized,
+ 'tags': tags,
+ **({} if metadata is None else metadata),
+ **({} if inputs is None else inputs),
+ **kwargs}
+ )
+
+ @debug_print_function_params
+ async def on_tool_end(
+ self,
+ output: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ tool_event: ToolEvent = self.events.tool[str(run_id)]
+ tool_event.end_timestamp = get_ISO_time()
+ tool_event.returns = output
+ self.ao_client.record(tool_event)
+
+ # Tools are capable of failing `on_tool_end` quietly.
+ # This is a workaround to make sure we can log it as an error.
+ if kwargs.get('name') == '_Exception':
+ error_event = ErrorEvent(trigger_event=tool_event, error_type="LangchainToolException", details=output)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ async def on_tool_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ tool_event: ToolEvent = self.events.tool[str(run_id)]
+ self.ao_client.record(tool_event)
+
+ error_event = ErrorEvent(trigger_event=tool_event, exception=error)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ async def on_retriever_start(
+ self,
+ serialized: Dict[str, Any],
+ query: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> None:
+ self.events.retriever[str(run_id)] = ActionEvent(
+ params={**serialized,
+ "query": query,
+ **({} if metadata is None else metadata),
+ ** kwargs},
+ action_type="retriever"
+ )
+
+ @debug_print_function_params
+ async def on_retriever_end(
+ self,
+ documents: Sequence[Document],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ action_event: ActionEvent = self.events.retriever[str(run_id)]
+ action_event.logs = documents # TODO: Adding this. Might want to add elsewhere e.g. params
+ action_event.end_timestamp = get_ISO_time()
+ self.ao_client.record(action_event)
+
+ @debug_print_function_params
+ async def on_retriever_error(
+ self,
+ error: BaseException,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ action_event: ActionEvent = self.events.retriever[str(run_id)]
+ self.ao_client.record(action_event)
+
+ error_event = ErrorEvent(trigger_event=action_event, exception=error)
+ self.ao_client.record(error_event)
+
+ @debug_print_function_params
+ async def on_agent_action(
+ self,
+ action: AgentAction,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ self.agent_actions[run_id].append(ActionEvent(
+ params={"action": action,
+ **kwargs},
+ action_type="agent"
+ ))
+
+ @debug_print_function_params
+ async def on_agent_finish(
+ self,
+ finish: AgentFinish,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ # Need to attach finish to some on_agent_action so just choosing the last one
+ self.agent_actions[run_id][-1].returns = finish.to_json()
+
+ for agentAction in self.agent_actions[run_id]:
+ self.ao_client.record(agentAction)
+
+ # TODO: Create a way for the end user to set this based on their conditions
+ # self.ao_client.end_session("Success") #TODO: calling end_session here causes "No current session"
+
+ @debug_print_function_params
+ async def on_text(
+ self,
+ text: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ pass
+
+ @debug_print_function_params
+ async def on_retry(
+ self,
+ retry_state: RetryCallState,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ **kwargs: Any,
+ ) -> Any:
+ action_event = ActionEvent(
+ params={**kwargs},
+ returns=str(retry_state),
+ action_type="retry",
+ # result="Indeterminate" # TODO: currently have no way of recording Indeterminate
+ )
+ self.ao_client.record(action_event)
+
+ @property
+ async def session_id(self):
+ return self.ao_client.current_session_id