Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
docs and example nits
Braelyn Boynton7f2028623f43
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 888899f1..6aef6124 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -9,17 +9,15 @@ from .helpers import get_ISO_timedef check_call_stack_for_agent_id() -> str | None:- # only looks at the last 2 frames otherwise it will retrieve- # whatever agent class was used lastfor frame_info in inspect.stack():- # Get the locals from the current frame+ # Look through the call stack for the class that called the LLMlocal_vars = frame_info.frame.f_localsfor var in local_vars.values():+ # We stop looking up the stack at main because after that we see global variablesif var == "__main__":return- # Instead of checking if var is a class, check if it's an instance with the _is_ao_agent attributeif hasattr(var, '_is_ao_agent') and getattr(var, '_is_ao_agent'):- logging.info('LLM call from agent named: ' + getattr(var, '_ao_agent_name'))+ logging.debug('LLM call from agent named: ' + getattr(var, '_ao_agent_name'))return getattr(var, '_ao_agent_id')return None@@ -78,8 +76,8 @@ class LlmTracker:self.client.record(self.event_stream)self.event_stream = Noneexcept Exception as e:- logging.info(- f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ logging.warning(+ f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif inspect.isasyncgen(response):@@ -135,8 +133,8 @@ class LlmTracker:))# Standard responseexcept Exception as e:- logging.error(- f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ logging.warning(+ f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")return response@@ -187,8 +185,8 @@ class LlmTracker:self.client.record(self.event_stream)self.event_stream = Noneexcept Exception as e:- logging.error(- f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ logging.warning(+ f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif isinstance(response, Stream):diff --git a/examples/multi_agent_example.ipynb b/examples/multi_agent_example.ipynbindex 6d34bfed..e1ed0ec0 100644--- a/examples/multi_agent_example.ipynb+++ b/examples/multi_agent_example.ipynb@@ -25,14 +25,10 @@"from IPython.display import display, Markdown"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:50.230763Z",- "start_time": "2024-03-08T20:31:49.991628Z"- }+ "collapsed": false},"id": "7c566fac57d3b6ce",- "execution_count": 1+ "execution_count": null},{"cell_type": "code",@@ -41,42 +37,26 @@"load_dotenv()\n","OPENAI_API_KEY = os.getenv('OPENAI_API_KEY', \"<your_openai_key>\")\n","AGENTOPS_API_KEY = os.getenv('AGENTOPS_API_KEY', \"<your_agentops_key>\")\n",- "logging.basicConfig(level=logging.INFO) # this will let us see that calls are assigned to an agent"+ "logging.basicConfig(level=logging.DEBUG) # this will let us see that calls are assigned to an agent"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:50.238834Z",- "start_time": "2024-03-08T20:31:50.231406Z"- }+ "collapsed": false},"id": "9f8c52496c04693",- "execution_count": 2+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "INFO:root:View info on this session at https://agentops.ai/dashboard?session_id=e9ecbebe-5aee-4130-b007-895539ac6d96\n"- ]- }- ],+ "outputs": [],"source": ["ao_client = agentops.Client(AGENTOPS_API_KEY)\n","openai_client = openai.Client()"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:50.318099Z",- "start_time": "2024-03-08T20:31:50.234493Z"- }+ "collapsed": false},"id": "af062552554d60ce",- "execution_count": 3+ "execution_count": null},{"cell_type": "markdown",@@ -107,14 +87,10 @@" return res.choices[0].message.content"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:50.318509Z",- "start_time": "2024-03-08T20:31:50.310816Z"- }+ "collapsed": false},"id": "727e3cc26ce3ec3",- "execution_count": 4+ "execution_count": null},{"cell_type": "code",@@ -124,128 +100,80 @@"engineer = EngineerAgent()"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:50.375686Z",- "start_time": "2024-03-08T20:31:50.313794Z"- }+ "collapsed": false},"id": "79b75d65de738522",- "execution_count": 5+ "execution_count": null},{- "cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",- "INFO:root:LLM call from agent named: engineer\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are a software engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.'}], 'temperature': 0.5} - skipping upload to AgentOps\n"- ]- }+ "cell_type": "markdown",+ "source": [+ "Now we have our agents and we tagged them with the `@track_agent` decorator. Any LLM calls that go through this class will now be tagged as agent calls in AgentOps.\n",+ "\n",+ "Lets use these agents!"],+ "metadata": {+ "collapsed": false+ },+ "id": "69dd3af9206308cc"+ },+ {+ "cell_type": "code",+ "outputs": [],"source": ["generated_func = engineer.completion(\"Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.\")"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:51.169919Z",- "start_time": "2024-03-08T20:31:50.376440Z"- }+ "collapsed": false},"id": "69e76061a626549",- "execution_count": 6+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "data": {- "text/plain": "<IPython.core.display.Markdown object>",- "text/markdown": "```python\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n```"- },- "metadata": {},- "output_type": "display_data"- }- ],+ "outputs": [],"source": ["display(Markdown('```python\\n' + generated_func + '\\n```'))"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:51.170415Z",- "start_time": "2024-03-08T20:31:51.165957Z"- }+ "collapsed": false},"id": "830b86dac47dceb3",- "execution_count": 7+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",- "INFO:root:LLM call from agent named: qa\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are a qa engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python unit test that test the following function: \\n def multiply_and_divide(num1, num2):\\n result = (num1 * num2) / 2\\n return result'}], 'temperature': 0.5} - skipping upload to AgentOps\n"- ]- }- ],+ "outputs": [],"source": ["generated_test = qa.completion(\"Write a python unit test that test the following function: \\n \" + generated_func)"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:53.504281Z",- "start_time": "2024-03-08T20:31:51.172400Z"- }+ "collapsed": false},"id": "63c9d0d457aee91a",- "execution_count": 8+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "data": {- "text/plain": "<IPython.core.display.Markdown object>",- "text/markdown": "```python\nimport unittest\n\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n\nclass TestMultiplyAndDivide(unittest.TestCase):\n\n def test_multiply_and_divide(self):\n self.assertEqual(multiply_and_divide(4, 2), 4)\n self.assertEqual(multiply_and_divide(5, 3), 7.5)\n self.assertEqual(multiply_and_divide(0, 10), 0)\n self.assertEqual(multiply_and_divide(-6, 3), -9)\n\nif __name__ == '__main__':\n unittest.main()\n```"- },- "metadata": {},- "output_type": "display_data"- }- ],+ "outputs": [],"source": ["display(Markdown('```python\\n' + generated_test + '\\n```'))"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:53.504737Z",- "start_time": "2024-03-08T20:31:53.501676Z"- }+ "collapsed": false},"id": "a88ffcbd2015d422",- "execution_count": 9+ "execution_count": null+ },+ {+ "cell_type": "markdown",+ "source": [+ "Perfect! It generated the code as expected, and in the DEBUG logs, you can see that the calls were made by agents named \"engineer\" and \"qa\"!"+ ],+ "metadata": {+ "collapsed": false+ },+ "id": "1bd312ed049a5511"},{"cell_type": "markdown",@@ -259,30 +187,7 @@},{"cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are not a tracked agent'}, {'role': 'user', 'content': 'Say hello'}]} - skipping upload to AgentOps\n"- ]- },- {- "data": {- "text/plain": "'Hello! How can I assist you today?'"- },- "execution_count": 10,- "metadata": {},- "output_type": "execute_result"- }- ],+ "outputs": [],"source": ["res = openai_client.chat.completions.create(\n"," model=\"gpt-3.5-turbo\",\n",@@ -292,14 +197,10 @@"res.choices[0].message.content"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-08T20:31:54.407458Z",- "start_time": "2024-03-08T20:31:53.506437Z"- }+ "collapsed": false},"id": "122e923cb07fd5f4",- "execution_count": 10+ "execution_count": null},{"cell_type": "markdown",