Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Tidying up

Howard Gilbd3f7a59ec65
diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.py
index af5023ae..fa581b98 100644
--- a/agentops/langchain_callback_handler.py
+++ b/agentops/langchain_callback_handler.py
@@ -1,12 +1,13 @@
-from typing import Dict, Any, List, Optional, Sequence
+from typing import Dict, Any, List, Optional, Sequence, Union
from collections import defaultdict
from uuid import UUID
from langchain_core.agents import AgentFinish, AgentAction
from langchain_core.outputs import LLMResult
from langchain_core.documents import Document
-from langchain_core.outputs import LLMResult
+from langchain_core.outputs import ChatGenerationChunk, GenerationChunk, LLMResult
from langchain.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler
+from langchain_core.messages import BaseMessage
from tenacity import RetryCallState
@@ -383,6 +384,33 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
# tags=tags # TODO
)
+ @debug_print_function_params
+ async def on_chat_model_start(
+ self,
+ serialized: Dict[str, Any],
+ messages: List[List[BaseMessage]],
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ metadata: Optional[Dict[str, Any]] = None,
+ **kwargs: Any,
+ ) -> Any:
+ pass
+
+ @debug_print_function_params
+ async def on_llm_new_token(
+ self,
+ token: str,
+ *,
+ chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ pass
+
@debug_print_function_params
async def on_llm_error(
self,
@@ -631,6 +659,18 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
# TODO: Create a way for the end user to set this based on their conditions
# self.ao_client.end_session("Success") #TODO: calling end_session here causes "No current session"
+ @debug_print_function_params
+ async def on_text(
+ self,
+ text: str,
+ *,
+ run_id: UUID,
+ parent_run_id: Optional[UUID] = None,
+ tags: Optional[List[str]] = None,
+ **kwargs: Any,
+ ) -> None:
+ pass
+
@debug_print_function_params
async def on_retry(
self,
diff --git a/examples/langchain_examples.ipynb b/examples/langchain_examples.ipynb
index 3298fb6e..2498faf7 100644
--- a/examples/langchain_examples.ipynb
+++ b/examples/langchain_examples.ipynb
@@ -14,7 +14,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"id": "initial_id",
"metadata": {
"ExecuteTime": {
@@ -27,7 +27,6 @@
"import os\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.agents import initialize_agent, AgentType\n",
- "from dotenv import load_dotenv\n",
"from langchain.agents import tool"
]
},
@@ -41,7 +40,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"id": "585f00bb186711a7",
"metadata": {
"ExecuteTime": {
@@ -64,7 +63,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 3,
"id": "1490411415d7317c",
"metadata": {
"ExecuteTime": {
@@ -72,8 +71,20 @@
"start_time": "2023-12-15T20:21:11.479154Z"
}
},
- "outputs": [],
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "True"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
+ "from dotenv import load_dotenv\n",
"load_dotenv()"
]
},
@@ -84,27 +95,61 @@
"source": [
"This is where AgentOps comes into play. Before creating our LLM instance via Langchain, first we'll create an instance of the AO LangchainCallbackHandler. After the handler is initialized, a session will be recorded automatically.\n",
"\n",
- "Pass in your API key, and optionally any tags to describe this session for easier lookup in the AO dashboard. You can also retrieve the `session_id` of the newly created session."
+ "Pass in your API key, and optionally any tags to describe this session for easier lookup in the AO dashboard."
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 4,
"id": "deec8da7-5b88-487e-bb07-e1ec79147b72",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/howardgil/Desktop/agentops/AgentOps-AI/agentops/env/lib/python3.12/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The class `langchain_community.chat_models.openai.ChatOpenAI` was deprecated in langchain-community 0.0.10 and will be removed in 0.2.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import ChatOpenAI`.\n",
+ " warn_deprecated(\n"
+ ]
+ }
+ ],
"source": [
"AGENTOPS_API_KEY = os.environ.get('AGENTOPS_API_KEY')\n",
"OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')\n",
"\n",
"agentops_handler = AgentOpsLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Langchain Example'])\n",
"\n",
- "print(\"Agent Ops session ID: \" + str(agentops_handler.session_id))\n",
"llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY,\n",
" callbacks=[agentops_handler],\n",
" model='gpt-3.5-turbo')"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "576e5f97",
+ "metadata": {},
+ "source": [
+ "You can also retrieve the `session_id` of the newly created session."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "5c4ef053",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Agent Ops session ID: 85cb6f74-833d-4659-bc08-cf70544b6542\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"Agent Ops session ID: \" + str(agentops_handler.session_id))"
+ ]
+ },
{
"cell_type": "markdown",
"id": "93aa09ec",
@@ -115,15 +160,18 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 6,
"id": "6abf26f9",
"metadata": {},
"outputs": [],
"source": [
"@tool\n",
- "def find_movie(term) -> str:\n",
+ "def find_movie(genre) -> str:\n",
" \"\"\"Find available movies\"\"\"\n",
- " return 'Citizen Kane'\n",
+ " if genre == 'drama':\n",
+ " return 'Dune 2'\n",
+ " else:\n",
+ " return 'Pineapple Express'\n",
"\n",
"\n",
"tools = [find_movie]"
@@ -139,7 +187,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 7,
"id": "71d56635-d0db-4362-b140-5072abee249f",
"metadata": {},
"outputs": [],
@@ -158,7 +206,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 8,
"id": "d538b20aa954ee80",
"metadata": {
"ExecuteTime": {
@@ -166,7 +214,16 @@
"start_time": "2023-12-15T20:21:12.351126Z"
}
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/howardgil/Desktop/agentops/AgentOps-AI/agentops/env/lib/python3.12/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `initialize_agent` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use Use new agent constructor methods like create_react_agent, create_json_agent, create_structured_chat_agent, etc. instead.\n",
+ " warn_deprecated(\n"
+ ]
+ }
+ ],
"source": [
"agent = initialize_agent(tools,\n",
" llm,\n",
@@ -178,14 +235,113 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 9,
"id": "6dfb127553751384",
"metadata": {
"scrolled": true
},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "\n",
+ "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/Users/howardgil/Desktop/agentops/AgentOps-AI/agentops/env/lib/python3.12/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `run` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n",
+ " warn_deprecated(\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[32;1m\u001b[1;3mQuestion: What comedies are playing?\n",
+ "Thought: We should use the find_movie tool to search for available comedy movies.\n",
+ "Action:\n",
+ "```\n",
+ "{\n",
+ " \"action\": \"find_movie\",\n",
+ " \"action_input\": \"comedy\"\n",
+ "}\n",
+ "```\n",
+ "\u001b[0m\n",
+ "Observation: \u001b[36;1m\u001b[1;3mPineapple Express\u001b[0m\n",
+ "Thought:\u001b[32;1m\u001b[1;3mWe should check if there are more comedy movies available.\n",
+ "Action: \n",
+ "```\n",
+ "{\n",
+ " \"action\": \"find_movie\",\n",
+ " \"action_input\": \"comedy\"\n",
+ "}\n",
+ "```\u001b[0m\n",
+ "Observation: \u001b[36;1m\u001b[1;3mPineapple Express\u001b[0m\n",
+ "Thought:\u001b[32;1m\u001b[1;3mCould not parse LLM output: I now know the final answer.\u001b[0m\n",
+ "Observation: Invalid or incomplete response\n",
+ "Thought:\u001b[32;1m\u001b[1;3mI apologize for the confusion in the previous interaction. Let's try to find comedy movies again.\n",
+ "\n",
+ "Question: What comedies are playing?\n",
+ "Thought: Let's use the find_movie tool to search for available comedy movies.\n",
+ "Action:\n",
+ "```\n",
+ "{\n",
+ " \"action\": \"find_movie\",\n",
+ " \"action_input\": \"comedy\"\n",
+ "}\n",
+ "```\n",
+ "\u001b[0m\n",
+ "Observation: \u001b[36;1m\u001b[1;3mPineapple Express\u001b[0m\n",
+ "Thought:\u001b[32;1m\u001b[1;3mI apologize for the confusion in the previous interactions. Let's make sure to get a clear list of comedy movies this time.\n",
+ "\n",
+ "Question: What comedies are playing?\n",
+ "Thought: Let's try using the find_movie tool once again to search for available comedy movies.\n",
+ "Action:\n",
+ "```\n",
+ "{\n",
+ " \"action\": \"find_movie\",\n",
+ " \"action_input\": \"comedy\"\n",
+ "}\n",
+ "```\n",
+ "\u001b[0m\n",
+ "Observation: \u001b[36;1m\u001b[1;3mPineapple Express\u001b[0m\n",
+ "Thought:\u001b[32;1m\u001b[1;3mI apologize for the technical issues we encountered. Let's try finding comedy movies one more time.\n",
+ "\n",
+ "Question: What comedies are playing?\n",
+ "Thought: Let's use the find_movie tool to search for available comedy movies.\n",
+ "Action:\n",
+ "```\n",
+ "{\n",
+ " \"action\": \"find_movie\",\n",
+ " \"action_input\": \"comedy\"\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "\u001b[0m\n",
+ "Observation: \u001b[36;1m\u001b[1;3mPineapple Express\u001b[0m\n",
+ "Thought:\u001b[32;1m\u001b[1;3mFinal Answer: Pineapple Express\u001b[0m\n",
+ "\n",
+ "\u001b[1m> Finished chain.\u001b[0m\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "'Pineapple Express'"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
"source": [
- "agent.run(\"What movies are playing?\", callbacks=[agentops_handler])"
+ "agent.run(\"What comedies are playing?\", callbacks=[agentops_handler])"
]
},
{
@@ -215,7 +371,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 10,
"id": "973f00a2-24da-48b3-adf7-97872d0d5afd",
"metadata": {},
"outputs": [],
@@ -223,23 +379,22 @@
"import os\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.agents import initialize_agent, AgentType\n",
- "from dotenv import load_dotenv\n",
"from langchain.agents import tool"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 11,
"id": "16d8136c-cac2-47a5-85d8-cc466089feea",
"metadata": {},
"outputs": [],
"source": [
- "from agentops.langchain_callback_handler import AsyncLangchainCallbackHandler"
+ "from agentops.langchain_callback_handler import AsyncLangchainCallbackHandler as AgentOpsAsyncLangchainCallbackHandler"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 12,
"id": "b5986ed0-02d0-4e5f-99cd-c1fdfd7e5f1c",
"metadata": {},
"outputs": [],
@@ -253,31 +408,43 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 13,
"id": "bdce6ac1-bdbc-46ba-a9f1-7fd384187016",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Agent Ops session ID: 85cb6f74-833d-4659-bc08-cf70544b6542\n"
+ ]
+ }
+ ],
"source": [
- "agentops_handler = AsyncLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Async Example'])\n",
+ "agentops_handler = AgentOpsAsyncLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Async Example'])\n",
"\n",
"llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY,\n",
" callbacks=[agentops_handler],\n",
" model='gpt-3.5-turbo')\n",
"\n",
- "print(\"Agent Ops session ID: \" + agentops_handler.session_id)"
+ "print(\"Agent Ops session ID: \" + str(await agentops_handler.session_id))"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 14,
"id": "d728c326-9a41-498b-bb64-9720577aac3e",
"metadata": {},
"outputs": [],
"source": [
"@tool\n",
- "def find_movie(term) -> str:\n",
+ "def find_movie(genre) -> str:\n",
" \"\"\"Find available movies\"\"\"\n",
- " return 'Citizen Kane'\n",
+ " if genre == 'drama':\n",
+ " return 'Dune 2'\n",
+ " else:\n",
+ " return 'Pineapple Express'\n",
+ "\n",
"\n",
"tools = [find_movie]\n",
"\n",
@@ -287,20 +454,23 @@
},
{
"cell_type": "code",
- "execution_count": null,
- "id": "5d01e2b2-42f1-45ed-8a9d-fdfe82154926",
- "metadata": {},
- "outputs": [],
- "source": [
- "print(\"Agent Ops session ID: \" + agentops_handler.session_id)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
+ "execution_count": 15,
"id": "30eddc8b-0c90-4d96-9f1f-45da38c0d984",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "ename": "RuntimeError",
+ "evalue": "asyncio.run() cannot be called from a running event loop",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[15], line 13\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;28;01masync\u001b[39;00m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrun_async\u001b[39m():\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mawait\u001b[39;00m agent\u001b[38;5;241m.\u001b[39mrun(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWhat comedies are playing?\u001b[39m\u001b[38;5;124m\"\u001b[39m, callbacks\u001b[38;5;241m=\u001b[39m[agentops_handler])\n\u001b[0;32m---> 13\u001b[0m \u001b[43masyncio\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrun_async\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m/opt/homebrew/Cellar/[email protected]/3.12.2_1/Frameworks/Python.framework/Versions/3.12/lib/python3.12/asyncio/runners.py:190\u001b[0m, in \u001b[0;36mrun\u001b[0;34m(main, debug, loop_factory)\u001b[0m\n\u001b[1;32m 161\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Execute the coroutine and return the result.\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \n\u001b[1;32m 163\u001b[0m \u001b[38;5;124;03mThis function runs the passed coroutine, taking care of\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;124;03m asyncio.run(main())\u001b[39;00m\n\u001b[1;32m 187\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 188\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m events\u001b[38;5;241m.\u001b[39m_get_running_loop() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 189\u001b[0m \u001b[38;5;66;03m# fail fast with short traceback\u001b[39;00m\n\u001b[0;32m--> 190\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[1;32m 191\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124masyncio.run() cannot be called from a running event loop\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 193\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m Runner(debug\u001b[38;5;241m=\u001b[39mdebug, loop_factory\u001b[38;5;241m=\u001b[39mloop_factory) \u001b[38;5;28;01mas\u001b[39;00m runner:\n\u001b[1;32m 194\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m runner\u001b[38;5;241m.\u001b[39mrun(main)\n",
+ "\u001b[0;31mRuntimeError\u001b[0m: asyncio.run() cannot be called from a running event loop"
+ ]
+ }
+ ],
"source": [
"agent = initialize_agent(\n",
" tools,\n",
@@ -311,7 +481,7 @@
" callbacks=[agentops_handler]\n",
")\n",
"\n",
- "response = await agent.arun('What movies are playing?')"
+ "await agent.arun('What comedies are playing?')"
]
},
{
@@ -329,14 +499,6 @@
"\n",
"![image.png](attachment:69f2121a-d437-4c09-bbbe-c76c9243ee19.png)"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ea82dfb7-7530-4b2b-807a-499f0815d72d",
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/tests/langchain_handlers/test_langchain_handler.py b/tests/langchain_handlers/test_langchain_handler.py
index b0373367..39b25f42 100644
--- a/tests/langchain_handlers/test_langchain_handler.py
+++ b/tests/langchain_handlers/test_langchain_handler.py
@@ -1,38 +1,23 @@
+import asyncio
import os
from langchain.chat_models import ChatOpenAI
from langchain.agents import initialize_agent, AgentType
from dotenv import load_dotenv
from langchain.agents import tool
-from agentops.langchain_callback_handler import LangchainCallbackHandler as AgentOpsLangchainCallbackHandler
-
+from agentops.langchain_callback_handler import LangchainCallbackHandler as AgentOpsLangchainCallbackHandler, AsyncLangchainCallbackHandler as AgentOpsAsyncLangchainCallbackHandler
load_dotenv()
-
AGENTOPS_API_KEY = os.environ.get('AGENTOPS_API_KEY')
OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')
agentops_handler = AgentOpsLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Langchain Example'])
-print("Agent Ops session ID: " + str(agentops_handler.session_id))
llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY,
callbacks=[agentops_handler],
model='gpt-3.5-turbo')
-# @tool
-# def find_movie(genre, actor) -> str:
-# """Find available movies"""
-# # raise ValueError("This is an intentional error for testing.")
-# if genre == 'drama':
-# if actor == 'Zendaya':
-# return 'Dune 2'
-# else:
-# return 'Citizen Kane'
-# else:
-# return 'Pineapple Express'
-
-
@tool
def find_movie(genre) -> str:
"""Find available movies"""
@@ -57,7 +42,26 @@ agent = initialize_agent(tools,
agent.run("What comedies are playing?", callbacks=[agentops_handler])
-# agent.run("What dramas are playing?", callbacks=[agentops_handler])
-# agent.run("What dramas are playing starring zendaya?", callbacks=[agentops_handler])
-# agent.run("What movies are playing starring zendaya?", callbacks=[agentops_handler])
+
+########
+# Async
+
+agentops_handler = AgentOpsAsyncLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Async Example'])
+
+llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY,
+ callbacks=[agentops_handler],
+ model='gpt-3.5-turbo')
+
+agent = initialize_agent(tools,
+ llm,
+ agent=AgentType.CHAT_ZERO_SHOT_REACT_DESCRIPTION,
+ verbose=True,
+ callbacks=[agentops_handler], # You must pass in a callback handler to record your agent
+ handle_parsing_errors=True)
+
+
+async def run_async():
+ await agent.run("What comedies are playing?", callbacks=[agentops_handler])
+
+asyncio.run(run_async())