Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

docs and example nits

Braelyn Boynton7f2028623f43
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
index 888899f1..6aef6124 100644
--- a/agentops/llm_tracker.py
+++ b/agentops/llm_tracker.py
@@ -9,17 +9,15 @@ from .helpers import get_ISO_time
def check_call_stack_for_agent_id() -> str | None:
- # only looks at the last 2 frames otherwise it will retrieve
- # whatever agent class was used last
for frame_info in inspect.stack():
- # Get the locals from the current frame
+ # Look through the call stack for the class that called the LLM
local_vars = frame_info.frame.f_locals
for var in local_vars.values():
+ # We stop looking up the stack at main because after that we see global variables
if var == "__main__":
return
- # Instead of checking if var is a class, check if it's an instance with the _is_ao_agent attribute
if hasattr(var, '_is_ao_agent') and getattr(var, '_is_ao_agent'):
- logging.info('LLM call from agent named: ' + getattr(var, '_ao_agent_name'))
+ logging.debug('LLM call from agent named: ' + getattr(var, '_ao_agent_name'))
return getattr(var, '_ao_agent_id')
return None
@@ -78,8 +76,8 @@ class LlmTracker:
self.client.record(self.event_stream)
self.event_stream = None
except Exception as e:
- logging.info(
- f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ logging.warning(
+ f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
if inspect.isasyncgen(response):
@@ -135,8 +133,8 @@ class LlmTracker:
))
# Standard response
except Exception as e:
- logging.error(
- f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ logging.warning(
+ f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
return response
@@ -187,8 +185,8 @@ class LlmTracker:
self.client.record(self.event_stream)
self.event_stream = None
except Exception as e:
- logging.error(
- f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ logging.warning(
+ f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
if isinstance(response, Stream):
diff --git a/examples/multi_agent_example.ipynb b/examples/multi_agent_example.ipynb
index 6d34bfed..e1ed0ec0 100644
--- a/examples/multi_agent_example.ipynb
+++ b/examples/multi_agent_example.ipynb
@@ -25,14 +25,10 @@
"from IPython.display import display, Markdown"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:50.230763Z",
- "start_time": "2024-03-08T20:31:49.991628Z"
- }
+ "collapsed": false
},
"id": "7c566fac57d3b6ce",
- "execution_count": 1
+ "execution_count": null
},
{
"cell_type": "code",
@@ -41,42 +37,26 @@
"load_dotenv()\n",
"OPENAI_API_KEY = os.getenv('OPENAI_API_KEY', \"<your_openai_key>\")\n",
"AGENTOPS_API_KEY = os.getenv('AGENTOPS_API_KEY', \"<your_agentops_key>\")\n",
- "logging.basicConfig(level=logging.INFO) # this will let us see that calls are assigned to an agent"
+ "logging.basicConfig(level=logging.DEBUG) # this will let us see that calls are assigned to an agent"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:50.238834Z",
- "start_time": "2024-03-08T20:31:50.231406Z"
- }
+ "collapsed": false
},
"id": "9f8c52496c04693",
- "execution_count": 2
+ "execution_count": null
},
{
"cell_type": "code",
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "INFO:root:View info on this session at https://agentops.ai/dashboard?session_id=e9ecbebe-5aee-4130-b007-895539ac6d96\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"ao_client = agentops.Client(AGENTOPS_API_KEY)\n",
"openai_client = openai.Client()"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:50.318099Z",
- "start_time": "2024-03-08T20:31:50.234493Z"
- }
+ "collapsed": false
},
"id": "af062552554d60ce",
- "execution_count": 3
+ "execution_count": null
},
{
"cell_type": "markdown",
@@ -107,14 +87,10 @@
" return res.choices[0].message.content"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:50.318509Z",
- "start_time": "2024-03-08T20:31:50.310816Z"
- }
+ "collapsed": false
},
"id": "727e3cc26ce3ec3",
- "execution_count": 4
+ "execution_count": null
},
{
"cell_type": "code",
@@ -124,128 +100,80 @@
"engineer = EngineerAgent()"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:50.375686Z",
- "start_time": "2024-03-08T20:31:50.313794Z"
- }
+ "collapsed": false
},
"id": "79b75d65de738522",
- "execution_count": 5
+ "execution_count": null
},
{
- "cell_type": "code",
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
- "INFO:root:LLM call from agent named: engineer\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are a software engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.'}], 'temperature': 0.5} - skipping upload to AgentOps\n"
- ]
- }
+ "cell_type": "markdown",
+ "source": [
+ "Now we have our agents and we tagged them with the `@track_agent` decorator. Any LLM calls that go through this class will now be tagged as agent calls in AgentOps.\n",
+ "\n",
+ "Lets use these agents!"
],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "69dd3af9206308cc"
+ },
+ {
+ "cell_type": "code",
+ "outputs": [],
"source": [
"generated_func = engineer.completion(\"Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.\")"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:51.169919Z",
- "start_time": "2024-03-08T20:31:50.376440Z"
- }
+ "collapsed": false
},
"id": "69e76061a626549",
- "execution_count": 6
+ "execution_count": null
},
{
"cell_type": "code",
- "outputs": [
- {
- "data": {
- "text/plain": "<IPython.core.display.Markdown object>",
- "text/markdown": "```python\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n```"
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
+ "outputs": [],
"source": [
"display(Markdown('```python\\n' + generated_func + '\\n```'))"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:51.170415Z",
- "start_time": "2024-03-08T20:31:51.165957Z"
- }
+ "collapsed": false
},
"id": "830b86dac47dceb3",
- "execution_count": 7
+ "execution_count": null
},
{
"cell_type": "code",
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
- "INFO:root:LLM call from agent named: qa\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are a qa engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python unit test that test the following function: \\n def multiply_and_divide(num1, num2):\\n result = (num1 * num2) / 2\\n return result'}], 'temperature': 0.5} - skipping upload to AgentOps\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"generated_test = qa.completion(\"Write a python unit test that test the following function: \\n \" + generated_func)"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:53.504281Z",
- "start_time": "2024-03-08T20:31:51.172400Z"
- }
+ "collapsed": false
},
"id": "63c9d0d457aee91a",
- "execution_count": 8
+ "execution_count": null
},
{
"cell_type": "code",
- "outputs": [
- {
- "data": {
- "text/plain": "<IPython.core.display.Markdown object>",
- "text/markdown": "```python\nimport unittest\n\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n\nclass TestMultiplyAndDivide(unittest.TestCase):\n\n def test_multiply_and_divide(self):\n self.assertEqual(multiply_and_divide(4, 2), 4)\n self.assertEqual(multiply_and_divide(5, 3), 7.5)\n self.assertEqual(multiply_and_divide(0, 10), 0)\n self.assertEqual(multiply_and_divide(-6, 3), -9)\n\nif __name__ == '__main__':\n unittest.main()\n```"
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
+ "outputs": [],
"source": [
"display(Markdown('```python\\n' + generated_test + '\\n```'))"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:53.504737Z",
- "start_time": "2024-03-08T20:31:53.501676Z"
- }
+ "collapsed": false
},
"id": "a88ffcbd2015d422",
- "execution_count": 9
+ "execution_count": null
+ },
+ {
+ "cell_type": "markdown",
+ "source": [
+ "Perfect! It generated the code as expected, and in the DEBUG logs, you can see that the calls were made by agents named \"engineer\" and \"qa\"!"
+ ],
+ "metadata": {
+ "collapsed": false
+ },
+ "id": "1bd312ed049a5511"
},
{
"cell_type": "markdown",
@@ -259,30 +187,7 @@
},
{
"cell_type": "code",
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are not a tracked agent'}, {'role': 'user', 'content': 'Say hello'}]} - skipping upload to AgentOps\n"
- ]
- },
- {
- "data": {
- "text/plain": "'Hello! How can I assist you today?'"
- },
- "execution_count": 10,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"res = openai_client.chat.completions.create(\n",
" model=\"gpt-3.5-turbo\",\n",
@@ -292,14 +197,10 @@
"res.choices[0].message.content"
],
"metadata": {
- "collapsed": false,
- "ExecuteTime": {
- "end_time": "2024-03-08T20:31:54.407458Z",
- "start_time": "2024-03-08T20:31:53.506437Z"
- }
+ "collapsed": false
},
"id": "122e923cb07fd5f4",
- "execution_count": 10
+ "execution_count": null
},
{
"cell_type": "markdown",