Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
standin
reibs2490a56fc332
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 523208eb..84d6a6a7 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -2,4 +2,4 @@ from .client import Clientfrom .event import Eventfrom .logger import AgentOpsLoggerfrom .helpers import Models-from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandler+from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandler, TestHandlerdiff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.pyindex 54954692..473b544f 100644--- a/agentops/langchain_callback_handler.py+++ b/agentops/langchain_callback_handler.py@@ -1,10 +1,14 @@-from typing import Dict, Any, List, Optional+from typing import Dict, Any, List, Optional, Unionfrom uuid import UUIDfrom langchain_core.agents import AgentFinish, AgentActionfrom langchain_core.outputs import LLMResultfrom langchain_core.documents import Document+from langchain_core.messages import BaseMessage+from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult++from agentops import Client as AOClientfrom agentops import Eventfrom tenacity import RetryCallState@@ -557,3 +561,260 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):@propertydef session_id(self):return self.ao_client.session.session_id+++class TestHandler(AsyncCallbackHandler):+ """Async callback handler that handles callbacks from LangChain."""++ async def on_llm_start(+ self,+ serialized: Dict[str, Any],+ prompts: List[str],+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ metadata: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when LLM starts running."""+ print(+ f"LLM start: run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")++ async def on_chat_model_start(+ self,+ serialized: Dict[str, Any],+ messages: List[List[BaseMessage]],+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ metadata: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> Any:+ """Run when a chat model starts running."""+ print(+ f"Chat model start: run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")+ raise NotImplementedError(+ f"{self.__class__.__name__} does not implement `on_chat_model_start`"+ )++ async def on_llm_new_token(+ self,+ token: str,+ *,+ chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run on new LLM token. Only available when streaming is enabled."""+ print(+ f"LLM new token: token={token}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_llm_end(+ self,+ response: LLMResult,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when LLM ends running."""+ print(+ f"LLM end: response={response}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_llm_error(+ self,+ error: BaseException,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when LLM errors.+ Args:+ error (BaseException): The error that occurred.+ kwargs (Any): Additional keyword arguments.+ - response (LLMResult): The response which was generated before+ the error occurred.+ """+ print(+ f"LLM error: error={error}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_chain_start(+ self,+ serialized: Dict[str, Any],+ inputs: Dict[str, Any],+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ metadata: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when chain starts running."""+ print(+ f"Chain start: serialized={serialized}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")++ async def on_chain_end(+ self,+ outputs: Dict[str, Any],+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when chain ends running."""+ print(+ f"Chain end: outputs={outputs}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_chain_error(+ self,+ error: BaseException,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when chain errors."""+ print(+ f"Chain error: error={error}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_tool_start(+ self,+ serialized: Dict[str, Any],+ input_str: str,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ metadata: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when tool starts running."""+ print(+ f"Tool start: serialized={serialized}, input_str={input_str}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")++ async def on_tool_end(+ self,+ output: str,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when tool ends running."""+ print(+ f"Tool end: output={output}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_tool_error(+ self,+ error: BaseException,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run when tool errors."""+ print(+ f"Tool error: error={error}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_text(+ self,+ text: str,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run on arbitrary text."""+ print(+ f"Text: text={text}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_retry(+ self,+ retry_state: RetryCallState,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ **kwargs: Any,+ ) -> Any:+ """Run on a retry event."""+ print(+ f"Retry: retry_state={retry_state}, run_id={run_id}, parent_run_id={parent_run_id}")++ async def on_agent_action(+ self,+ action: AgentAction,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run on agent action."""+ print(+ f"Agent action: action={action}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_agent_finish(+ self,+ finish: AgentFinish,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run on agent end."""+ print(+ f"Agent finish: finish={finish}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_retriever_start(+ self,+ serialized: Dict[str, Any],+ query: str,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ metadata: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> None:+ """Run on retriever start."""+ print(+ f"Retriever start: serialized={serialized}, query={query}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")++ async def on_retriever_end(+ self,+ documents: Sequence[Document],+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run on retriever end."""+ print(+ f"Retriever end: documents={documents}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")++ async def on_retriever_error(+ self,+ error: BaseException,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ """Run on retriever error."""+ print(+ f"Retriever error: error={error}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")