Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
removed debugs, serialized results
reibs78a9f7ab9b35
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 84d6a6a7..523208eb 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -2,4 +2,4 @@ from .client import Clientfrom .event import Eventfrom .logger import AgentOpsLoggerfrom .helpers import Models-from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandler, TestHandler+from .langchain_callback_handler import LangchainCallbackHandler, AsyncLangchainCallbackHandlerdiff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.pyindex 07e52d60..0a1c3aef 100644--- a/agentops/langchain_callback_handler.py+++ b/agentops/langchain_callback_handler.py@@ -82,7 +82,6 @@ class LangchainCallbackHandler(BaseCallbackHandler):self.events[run_id].result = "Fail"self.ao_client.record(self.events[run_id])- print('llm ended and emitted')# Chain callbacksdef on_chain_start(@@ -144,7 +143,6 @@ class LangchainCallbackHandler(BaseCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> Any:- print('sync tool use')"""Run when tool starts running."""self.events[run_id] = Event(event_type="tool",@@ -259,7 +257,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):"""Run on agent finish."""self.events[run_id].end_timestamp = get_ISO_time()self.events[run_id].result = "Success"- self.events[run_id].returns = finish+ self.events[run_id].returns = finish.to_json()self.ao_client.record(self.events[run_id])@@ -367,7 +365,6 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):metadata: Optional[Dict[str, Any]] = None,**kwargs: Any,) -> Any:- print('on_chain_start')self.events[run_id] = Event(event_type="chain",init_timestamp=get_ISO_time(),@@ -385,7 +382,6 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- print('on_chain_end')self.events[run_id].end_timestamp = get_ISO_time()self.events[run_id].result = "Success"self.events[run_id].returns = outputs@@ -400,7 +396,6 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- print('on_chain_error')self.events[run_id].end_timestamp = get_ISO_time()self.events[run_id].result = "Fail"self.events[run_id].returns = str(error)@@ -420,7 +415,6 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):**kwargs: Any,) -> Any:"""Run when tool starts running."""- print('on_tool_start')self.events[run_id] = Event(event_type="tool",init_timestamp=get_ISO_time(),@@ -436,7 +430,6 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):parent_run_id: Optional[UUID] = None,**kwargs: Any,) -> Any:- print('on_tool_end')# Tools are capable of failing `on_tool_end` quietly.# This is a workaround to make sure we can log it as an error.if kwargs.get('name') == '_Exception':@@ -535,7 +528,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):"""Run on agent finish."""self.events[run_id].end_timestamp = get_ISO_time()self.events[run_id].result = "Success"- self.events[run_id].returns = finish+ self.events[run_id].returns = finish.to_json()self.ao_client.record(self.events[run_id])@@ -565,260 +558,3 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):@propertydef session_id(self):return self.ao_client.session.session_id---class TestHandler(AsyncCallbackHandler):- """Async callback handler that handles callbacks from LangChain."""-- async def on_llm_start(- self,- serialized: Dict[str, Any],- prompts: List[str],- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- metadata: Optional[Dict[str, Any]] = None,- **kwargs: Any,- ) -> None:- """Run when LLM starts running."""- print(- f"LLM start: run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")-- async def on_chat_model_start(- self,- serialized: Dict[str, Any],- messages: List[List[BaseMessage]],- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- metadata: Optional[Dict[str, Any]] = None,- **kwargs: Any,- ) -> Any:- """Run when a chat model starts running."""- print(- f"Chat model start: run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")- raise NotImplementedError(- f"{self.__class__.__name__} does not implement `on_chat_model_start`"- )-- async def on_llm_new_token(- self,- token: str,- *,- chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run on new LLM token. Only available when streaming is enabled."""- print(- f"LLM new token: token={token}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_llm_end(- self,- response: LLMResult,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run when LLM ends running."""- print(- f"LLM end: response={response}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_llm_error(- self,- error: BaseException,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run when LLM errors.- Args:- error (BaseException): The error that occurred.- kwargs (Any): Additional keyword arguments.- - response (LLMResult): The response which was generated before- the error occurred.- """- print(- f"LLM error: error={error}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_chain_start(- self,- serialized: Dict[str, Any],- inputs: Dict[str, Any],- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- metadata: Optional[Dict[str, Any]] = None,- **kwargs: Any,- ) -> None:- """Run when chain starts running."""- print(- f"Chain start: serialized={serialized}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")-- async def on_chain_end(- self,- outputs: Dict[str, Any],- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run when chain ends running."""- print(- f"Chain end: outputs={outputs}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_chain_error(- self,- error: BaseException,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run when chain errors."""- print(- f"Chain error: error={error}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_tool_start(- self,- serialized: Dict[str, Any],- input_str: str,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- metadata: Optional[Dict[str, Any]] = None,- **kwargs: Any,- ) -> None:- """Run when tool starts running."""- print(- f"Tool start: serialized={serialized}, input_str={input_str}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")-- async def on_tool_end(- self,- output: str,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run when tool ends running."""- print(- f"Tool end: output={output}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_tool_error(- self,- error: BaseException,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run when tool errors."""- print(- f"Tool error: error={error}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_text(- self,- text: str,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run on arbitrary text."""- print(- f"Text: text={text}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_retry(- self,- retry_state: RetryCallState,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- **kwargs: Any,- ) -> Any:- """Run on a retry event."""- print(- f"Retry: retry_state={retry_state}, run_id={run_id}, parent_run_id={parent_run_id}")-- async def on_agent_action(- self,- action: AgentAction,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run on agent action."""- print(- f"Agent action: action={action}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_agent_finish(- self,- finish: AgentFinish,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run on agent end."""- print(- f"Agent finish: finish={finish}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_retriever_start(- self,- serialized: Dict[str, Any],- query: str,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- metadata: Optional[Dict[str, Any]] = None,- **kwargs: Any,- ) -> None:- """Run on retriever start."""- print(- f"Retriever start: serialized={serialized}, query={query}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}, metadata={metadata}")-- async def on_retriever_end(- self,- documents: Sequence[Document],- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run on retriever end."""- print(- f"Retriever end: documents={documents}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")-- async def on_retriever_error(- self,- error: BaseException,- *,- run_id: UUID,- parent_run_id: Optional[UUID] = None,- tags: Optional[List[str]] = None,- **kwargs: Any,- ) -> None:- """Run on retriever error."""- print(- f"Retriever error: error={error}, run_id={run_id}, parent_run_id={parent_run_id}, tags={tags}")