Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

cleaned up notebook example

Braelyn Boynton3430a11457a4
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
index d6bd1f70..888899f1 100644
--- a/agentops/llm_tracker.py
+++ b/agentops/llm_tracker.py
@@ -135,7 +135,7 @@ class LlmTracker:
))
# Standard response
except Exception as e:
- print(
+ logging.error(
f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
return response
@@ -187,7 +187,7 @@ class LlmTracker:
self.client.record(self.event_stream)
self.event_stream = None
except Exception as e:
- print(
+ logging.error(
f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
@@ -234,7 +234,7 @@ class LlmTracker:
))
# Standard response
except Exception as e:
- print(
+ logging.error(
f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
return response
diff --git a/examples/multi_agent_example.ipynb b/examples/multi_agent_example.ipynb
index d7ef41b4..6d34bfed 100644
--- a/examples/multi_agent_example.ipynb
+++ b/examples/multi_agent_example.ipynb
@@ -21,15 +21,14 @@
"import os\n",
"import openai\n",
"import logging\n",
- "import traceback\n",
"\n",
"from IPython.display import display, Markdown"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:22.647026Z",
- "start_time": "2024-03-08T20:18:22.400023Z"
+ "end_time": "2024-03-08T20:31:50.230763Z",
+ "start_time": "2024-03-08T20:31:49.991628Z"
}
},
"id": "7c566fac57d3b6ce",
@@ -42,13 +41,13 @@
"load_dotenv()\n",
"OPENAI_API_KEY = os.getenv('OPENAI_API_KEY', \"<your_openai_key>\")\n",
"AGENTOPS_API_KEY = os.getenv('AGENTOPS_API_KEY', \"<your_agentops_key>\")\n",
- "logging.basicConfig(level=logging.INFO)"
+ "logging.basicConfig(level=logging.INFO) # this will let us see that calls are assigned to an agent"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:22.647457Z",
- "start_time": "2024-03-08T20:18:22.642757Z"
+ "end_time": "2024-03-08T20:31:50.238834Z",
+ "start_time": "2024-03-08T20:31:50.231406Z"
}
},
"id": "9f8c52496c04693",
@@ -61,19 +60,19 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "INFO:root:View info on this session at https://agentops.ai/dashboard?session_id=5193a9a7-aa7f-485d-843b-36518566e983\n"
+ "INFO:root:View info on this session at https://agentops.ai/dashboard?session_id=e9ecbebe-5aee-4130-b007-895539ac6d96\n"
]
}
],
"source": [
- "ao_client = agentops.Client(AGENTOPS_API_KEY, endpoint='http://localhost:8000')\n",
+ "ao_client = agentops.Client(AGENTOPS_API_KEY)\n",
"openai_client = openai.Client()"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:22.714693Z",
- "start_time": "2024-03-08T20:18:22.646071Z"
+ "end_time": "2024-03-08T20:31:50.318099Z",
+ "start_time": "2024-03-08T20:31:50.234493Z"
}
},
"id": "af062552554d60ce",
@@ -89,52 +88,6 @@
},
"id": "95d212546aaf1f82"
},
- {
- "cell_type": "code",
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are not a tracked agent'}, {'role': 'user', 'content': 'Say hello'}]} - skipping upload to AgentOps\n"
- ]
- },
- {
- "data": {
- "text/plain": "'Hello! How can I assist you today?'"
- },
- "execution_count": 12,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "res = openai_client.chat.completions.create(\n",
- " model=\"gpt-3.5-turbo\",\n",
- " messages=[{\"role\": \"system\", \"content\": \"You are not a tracked agent\"},\n",
- " {\"role\": \"user\", \"content\": \"Say hello\"}]\n",
- ")\n",
- "# traceback.print_exc()\n",
- "# raise Exception()\n",
- "res.choices[0].message.content"
- ],
- "metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:18:31.386272Z",
- "start_time": "2024-03-08T20:18:31.060728Z"
- }
- },
- "id": "bf2875d35c8458ae",
- "execution_count": 12
- },
{
"cell_type": "code",
"outputs": [],
@@ -156,58 +109,29 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:23.471733Z",
- "start_time": "2024-03-08T20:18:23.466899Z"
+ "end_time": "2024-03-08T20:31:50.318509Z",
+ "start_time": "2024-03-08T20:31:50.310816Z"
}
},
"id": "727e3cc26ce3ec3",
- "execution_count": 5
+ "execution_count": 4
},
{
"cell_type": "code",
"outputs": [],
"source": [
"qa = QaAgent()\n",
- "qa2 = QaAgent()\n",
"engineer = EngineerAgent()"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:23.565195Z",
- "start_time": "2024-03-08T20:18:23.468586Z"
+ "end_time": "2024-03-08T20:31:50.375686Z",
+ "start_time": "2024-03-08T20:31:50.313794Z"
}
},
"id": "79b75d65de738522",
- "execution_count": 6
- },
- {
- "cell_type": "code",
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "engineer\n",
- "fc947d29-c909-4470-bd33-f0da4687f81a\n",
- "3ecfe8d2-df1a-4189-85b1-0736d4314843\n"
- ]
- }
- ],
- "source": [
- "print(engineer._ao_agent_name)\n",
- "print(qa._ao_agent_id)\n",
- "print(qa2._ao_agent_id)"
- ],
- "metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:18:23.566003Z",
- "start_time": "2024-03-08T20:18:23.563967Z"
- }
- },
- "id": "513819204dde0e0a",
- "execution_count": 7
+ "execution_count": 5
},
{
"cell_type": "code",
@@ -234,12 +158,12 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:24.427146Z",
- "start_time": "2024-03-08T20:18:23.566776Z"
+ "end_time": "2024-03-08T20:31:51.169919Z",
+ "start_time": "2024-03-08T20:31:50.376440Z"
}
},
"id": "69e76061a626549",
- "execution_count": 8
+ "execution_count": 6
},
{
"cell_type": "code",
@@ -254,17 +178,17 @@
}
],
"source": [
- "display(Markdown('```python\\n' + generated_func + '\\n```'))\n"
+ "display(Markdown('```python\\n' + generated_func + '\\n```'))"
],
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:24.427763Z",
- "start_time": "2024-03-08T20:18:24.424968Z"
+ "end_time": "2024-03-08T20:31:51.170415Z",
+ "start_time": "2024-03-08T20:31:51.165957Z"
}
},
"id": "830b86dac47dceb3",
- "execution_count": 9
+ "execution_count": 7
},
{
"cell_type": "code",
@@ -291,12 +215,12 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:27.148493Z",
- "start_time": "2024-03-08T20:18:24.429637Z"
+ "end_time": "2024-03-08T20:31:53.504281Z",
+ "start_time": "2024-03-08T20:31:51.172400Z"
}
},
"id": "63c9d0d457aee91a",
- "execution_count": 10
+ "execution_count": 8
},
{
"cell_type": "code",
@@ -304,7 +228,7 @@
{
"data": {
"text/plain": "<IPython.core.display.Markdown object>",
- "text/markdown": "```python\nimport unittest\n\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n\nclass TestMultiplyAndDivide(unittest.TestCase):\n\n def test_multiply_and_divide(self):\n self.assertEqual(multiply_and_divide(4, 2), 4.0)\n self.assertEqual(multiply_and_divide(5, 3), 7.5)\n self.assertEqual(multiply_and_divide(0, 10), 0.0)\n self.assertEqual(multiply_and_divide(-6, 3), -9.0)\n\nif __name__ == '__main__':\n unittest.main()\n```"
+ "text/markdown": "```python\nimport unittest\n\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n\nclass TestMultiplyAndDivide(unittest.TestCase):\n\n def test_multiply_and_divide(self):\n self.assertEqual(multiply_and_divide(4, 2), 4)\n self.assertEqual(multiply_and_divide(5, 3), 7.5)\n self.assertEqual(multiply_and_divide(0, 10), 0)\n self.assertEqual(multiply_and_divide(-6, 3), -9)\n\nif __name__ == '__main__':\n unittest.main()\n```"
},
"metadata": {},
"output_type": "display_data"
@@ -316,26 +240,76 @@
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:27.151987Z",
- "start_time": "2024-03-08T20:18:27.148793Z"
+ "end_time": "2024-03-08T20:31:53.504737Z",
+ "start_time": "2024-03-08T20:31:53.501676Z"
}
},
"id": "a88ffcbd2015d422",
- "execution_count": 11
+ "execution_count": 9
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "Lets verify one more thing! If we make an LLM call outside of the context of a tracked agent, we want to make sure it gets assigned to the Default Agent."
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "cbd0817a31756397"
},
{
"cell_type": "code",
- "outputs": [],
- "source": [],
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are not a tracked agent'}, {'role': 'user', 'content': 'Say hello'}]} - skipping upload to AgentOps\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": "'Hello! How can I assist you today?'"
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "res = openai_client.chat.completions.create(\n",
+ " model=\"gpt-3.5-turbo\",\n",
+ " messages=[{\"role\": \"system\", \"content\": \"You are not a tracked agent\"},\n",
+ " {\"role\": \"user\", \"content\": \"Say hello\"}]\n",
+ ")\n",
+ "res.choices[0].message.content"
+ ],
"metadata": {
"collapsed": false,
"ExecuteTime": {
- "end_time": "2024-03-08T20:18:27.155679Z",
- "start_time": "2024-03-08T20:18:27.151979Z"
+ "end_time": "2024-03-08T20:31:54.407458Z",
+ "start_time": "2024-03-08T20:31:53.506437Z"
}
},
- "id": "e5003033f93eb21b",
- "execution_count": 11
+ "id": "122e923cb07fd5f4",
+ "execution_count": 10
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "You'll notice that we didn't log an agent name, so the AgentOps backend will assign it to the Default Agent for the session!"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "a30909020c6a1ada"
}
],
"metadata": {