Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
added support for all callbacks
Braelyn Boynton64ae928a4495
diff --git a/agentops/event.py b/agentops/event.pyindex 1bc0b2ab..50edbfca 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -4,6 +4,8 @@ AgentOps events.Classes:Event: Represents discrete events to be recorded."""+import json+from .helpers import get_ISO_time, Modelsfrom typing import Optional, Listfrom pydantic import Field@@ -32,7 +34,7 @@ class Event:event_type (str): Type of the event.params (str, optional): The parameters passed to the operation.returns (str, optional): The output of the operation.- result (str): Result of the operation.+ result (Result): Result of the operation as Enum Result.action_type (str): Type of action of the event.model (Models, optional): The model used during the event.prompt (str, optional): The input prompt for an LLM call.@@ -65,3 +67,16 @@ class Event:self.prompt = promptself.end_timestamp = get_ISO_time()self.init_timestamp = init_timestamp if init_timestamp else self.end_timestamp++ def __str__(self):+ return str({+ "event_type": self.event_type,+ "params": self.params,+ "returns": self.returns,+ "action_type": self.action_type,+ "result": self.result,+ "model": self.model,+ "prompt": self.prompt,+ "tags": self.tags,+ "init_timestamp": self.init_timestamp+ })diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.pyindex 313202ef..efd2cacf 100644--- a/agentops/langchain_callback_handler.py+++ b/agentops/langchain_callback_handler.py@@ -1,14 +1,17 @@from typing import Dict, Any, List, Optionalfrom uuid import UUID+from langchain_core.agents import AgentFinishfrom langchain_core.outputs import LLMResult+from langchain_core.documents import Documentfrom agentops import Client as AOClientfrom agentops import Event-from langchain.callbacks.base import BaseCallbackHandler+from langchain.callbacks.base import BaseCallbackHandler, ChainManagerMixinfrom agentops.helpers import get_ISO_time+from typing import Any, Dict, List, Optional, Sequenceclass LangchainCallbackHandler(BaseCallbackHandler):@@ -20,6 +23,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):# keypair <run_id: str, Event>self.events = {}+ # LLM Callbacksdef on_llm_start(self,serialized: Dict[str, Any],@@ -38,6 +42,19 @@ class LangchainCallbackHandler(BaseCallbackHandler):init_timestamp=get_ISO_time())+ def on_llm_error(+ self,+ error: BaseException,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ **kwargs: Any,+ ) -> Any:+ self.events[run_id].end_timestamp = get_ISO_time()+ self.events[run_id].result = "Fail"++ self.ao_client.record(self.events[run_id])+def on_llm_end(self,response: LLMResult,@@ -47,8 +64,140 @@ class LangchainCallbackHandler(BaseCallbackHandler):**kwargs: Any,) -> Any:self.events[run_id].end_timestamp = get_ISO_time()++ if len(response.generations) > 0:+ self.events[run_id].result = "Success"+ else:+ self.events[run_id].result = "Fail"++ self.ao_client.record(self.events[run_id])++ # Chain callbacks+ def on_chain_start(+ self,+ outputs: Dict[str, Any],+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ **kwargs: Any,+ ) -> Any:+ self.events[run_id] = Event(+ event_type="langchain_llm_chain",+ init_timestamp=get_ISO_time()+ )++ def on_chain_end(+ self,+ outputs: Dict[str, Any],+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ **kwargs: Any,+ ) -> Any:+ self.events[run_id].end_timestamp = get_ISO_time()+ self.events[run_id].result = "Success"++ self.ao_client.record(self.events[run_id])++ def on_chain_error(+ self,+ error: BaseException,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ **kwargs: Any,+ ) -> Any:+ self.events[run_id].end_timestamp = get_ISO_time()+ self.events[run_id].result = "Fail"+self.ao_client.record(self.events[run_id])+ # Tool callbacks+ def on_tool_end(+ self,+ output: str,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ **kwargs: Any,+ ) -> Any:+ self.ao_client.record(Event(+ event_type="langchain_tool_usage",+ result="Success",+ init_timestamp=get_ISO_time()+ ))++ def on_tool_error(+ self,+ error: BaseException,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ **kwargs: Any,+ ) -> Any:+ self.ao_client.record(Event(+ event_type="langchain_tool_usage",+ result="Fail",+ init_timestamp=get_ISO_time()+ ))++ # Retriever callbacks+ async def on_retriever_start(+ self,+ serialized: Dict[str, Any],+ query: str,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ metadata: Optional[Dict[str, Any]] = None,+ **kwargs: Any,+ ) -> None:+ self.events[run_id] = Event(+ event_type="langchain_llm_retriever",+ init_timestamp=get_ISO_time()+ )++ async def on_retriever_end(+ self,+ documents: Sequence[Document],+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ self.events[run_id].end_timestamp = get_ISO_time()+ self.events[run_id].result = "Successful"++ self.ao_client.record(self.events[run_id])++ async def on_retriever_error(+ self,+ error: BaseException,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ tags: Optional[List[str]] = None,+ **kwargs: Any,+ ) -> None:+ self.events[run_id].end_timestamp = get_ISO_time()+ self.events[run_id].result = "Fail"++ self.ao_client.record(self.events[run_id])++ # Agent callbacks+ def on_agent_finish(+ self,+ finish: AgentFinish,+ *,+ run_id: UUID,+ parent_run_id: Optional[UUID] = None,+ **kwargs: Any,+ ) -> Any:+ # TODO: Create a way for the end user to set this based on their conditions+ self.ao_client.end_session("Success")+@propertydef session_id(self):return self.ao_client.session.session_iddiff --git a/examples/langchain_examples.ipynb b/examples/langchain_examples.ipynbindex 8ff65714..077ea331 100644--- a/examples/langchain_examples.ipynb+++ b/examples/langchain_examples.ipynb@@ -21,8 +21,8 @@"metadata": {"collapsed": true,"ExecuteTime": {- "end_time": "2023-12-15T18:00:15.853166Z",- "start_time": "2023-12-15T18:00:14.651483Z"+ "end_time": "2023-12-15T20:21:11.477270Z",+ "start_time": "2023-12-15T20:21:10.289895Z"}},"outputs": [@@ -30,7 +30,7 @@"name": "stderr","output_type": "stream","text": [- "/Library/Python/3.9/site-packages/urllib3/__init__.py:34: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",+ "/Users/braelynboynton/Developer/agentops/venv/lib/python3.9/site-packages/urllib3/__init__.py:34: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n"," warnings.warn(\n"]}@@ -62,8 +62,8 @@"metadata": {"collapsed": false,"ExecuteTime": {- "end_time": "2023-12-15T18:00:15.861201Z",- "start_time": "2023-12-15T18:00:15.854596Z"+ "end_time": "2023-12-15T20:21:11.478111Z",+ "start_time": "2023-12-15T20:21:11.471462Z"}},"id": "585f00bb186711a7"@@ -90,8 +90,8 @@"metadata": {"collapsed": false,"ExecuteTime": {- "end_time": "2023-12-15T18:00:15.865514Z",- "start_time": "2023-12-15T18:00:15.861478Z"+ "end_time": "2023-12-15T20:21:11.494019Z",+ "start_time": "2023-12-15T20:21:11.479154Z"}},"id": "1490411415d7317c"@@ -116,7 +116,7 @@"name": "stdout","output_type": "stream","text": [- "Agent Ops session ID: 3c9deaa9-577c-49ac-9783-e7d0b0fecfbd\n"+ "Agent Ops session ID: 0feeb023-d8eb-4eae-9c77-29c2a3c88b38\n"]}],@@ -128,8 +128,8 @@"metadata": {"collapsed": false,"ExecuteTime": {- "end_time": "2023-12-15T18:00:16.808358Z",- "start_time": "2023-12-15T18:00:15.865350Z"+ "end_time": "2023-12-15T20:21:12.346995Z",+ "start_time": "2023-12-15T20:21:11.483591Z"}},"id": "432921383f39c9d5"@@ -137,7 +137,7 @@{"cell_type": "markdown","source": [- "Finally, let's use our agent! All of the actions will be recorded in the AO Dashboard"+ "Finally, let's use our agent! All the actions will be recorded in the AO Dashboard"],"metadata": {"collapsed": false@@ -158,74 +158,21 @@"metadata": {"collapsed": false,"ExecuteTime": {- "end_time": "2023-12-15T18:00:16.812313Z",- "start_time": "2023-12-15T18:00:16.809585Z"+ "end_time": "2023-12-15T20:21:12.352862Z",+ "start_time": "2023-12-15T20:21:12.351126Z"}},"id": "d538b20aa954ee80"},{"cell_type": "code",- "execution_count": 6,- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "\n",- "\u001B[1m> Entering new AgentExecutor chain...\u001B[0m\n",- "\u001B[32;1m\u001B[1;3mSure! Here's a short poem about secret agents:\n",- "\n",- "In shadows they dwell, unseen by the eye,\n",- "Masters of disguise, their secrets held high.\n",- "They weave through the night, like whispers in air,\n",- "Agents of mystery, with skills beyond compare.\n",- "\n",- "With steady hands, they handle the unknown,\n",- "Unraveling secrets, their cover not blown.\n",- "They traverse the globe, in search of the truth,\n",- "Stealthily moving, like shadows of youth.\n",- "\n",- "Their mission, you ask? To protect and defend,\n",- "The world's safety, until the very end.\n",- "With gadgets and wit, they outsmart their foes,\n",- "Silent warriors, where nobody knows.\n",- "\n",- "In dark alleys they gather, plans tightly knit,\n",- "Their loyalty unwavering, they never quit.\n",- "For justice they fight, with honor and grace,\n",- "A secret agent's life, a dangerous embrace.\n",- "\n",- "So raise your glass to these guardians unseen,\n",- "The heroes of shadows, in suits so pristine.\n",- "They sacrifice all, for a world that's unknown,\n",- "Secret agents, silent heroes, never alone.\n",- "\n",- "Final Answer: Secret agents, guardians of the night,\n",- "A poem to honor their courage and might.\u001B[0m\n",- "\n",- "\u001B[1m> Finished chain.\u001B[0m\n"- ]- },- {- "data": {- "text/plain": "'Secret agents, guardians of the night,\\nA poem to honor their courage and might.'"- },- "execution_count": 6,- "metadata": {},- "output_type": "execute_result"- }- ],+ "execution_count": null,+ "outputs": [],"source": ["agent.run(\"Please write me a short poem about secret agents\")"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2023-12-15T18:00:22.740852Z",- "start_time": "2023-12-15T18:00:16.812664Z"- }+ "collapsed": false},"id": "6dfb127553751384"}