Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
supports functions
Braelyn Boynton37cf12acc4e1
diff --git a/examples/multi_agent_example.ipynb b/examples/multi_agent_example.ipynbindex 61638a27..e1ed0ec0 100644--- a/examples/multi_agent_example.ipynb+++ b/examples/multi_agent_example.ipynb@@ -25,14 +25,10 @@"from IPython.display import display, Markdown"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:39.813046Z",- "start_time": "2024-03-09T00:22:39.580341Z"- }+ "collapsed": false},"id": "7c566fac57d3b6ce",- "execution_count": 1+ "execution_count": null},{"cell_type": "code",@@ -44,51 +40,23 @@"logging.basicConfig(level=logging.DEBUG) # this will let us see that calls are assigned to an agent"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:39.820790Z",- "start_time": "2024-03-09T00:22:39.814133Z"- }+ "collapsed": false},"id": "9f8c52496c04693",- "execution_count": 2+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): localhost:8000\n",- "DEBUG:urllib3.connectionpool:http://localhost:8000 \"POST /sessions HTTP/1.1\" 500 35\n",- "WARNING:root:AgentOps: Could not post data - internal server error\n",- "INFO:root:View info on this session at https://agentops.ai/dashboard?session_id=749afbc4-2df6-4a7b-bbdb-a2c5b9d745c5\n",- "DEBUG:httpx:load_ssl_context verify=True cert=None trust_env=True http2=False\n",- "DEBUG:httpx:load_verify_locations cafile='/opt/homebrew/lib/python3.11/site-packages/certifi/cacert.pem'\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "{'message': 'Internal Server Error'}\n"- ]- }- ],+ "outputs": [],"source": [- "ao_client = agentops.Client(AGENTOPS_API_KEY, endpoint=\"http://localhost:8000\")\n",+ "ao_client = agentops.Client(AGENTOPS_API_KEY)\n","openai_client = openai.Client()"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:39.945509Z",- "start_time": "2024-03-09T00:22:39.817178Z"- }+ "collapsed": false},"id": "af062552554d60ce",- "execution_count": 3+ "execution_count": null},{"cell_type": "markdown",@@ -119,52 +87,23 @@" return res.choices[0].message.content"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:39.951283Z",- "start_time": "2024-03-09T00:22:39.947705Z"- }+ "collapsed": false},"id": "727e3cc26ce3ec3",- "execution_count": 4+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): localhost:8000\n",- "DEBUG:urllib3.connectionpool:http://localhost:8000 \"POST /agents HTTP/1.1\" 500 35\n",- "WARNING:root:AgentOps: Could not post data - internal server error\n",- "DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): localhost:8000\n",- "DEBUG:urllib3.connectionpool:http://localhost:8000 \"POST /agents HTTP/1.1\" 500 35\n",- "WARNING:root:AgentOps: Could not post data - internal server error\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "{'message': 'Internal Server Error'}\n",- "{'message': 'Internal Server Error'}\n"- ]- }- ],+ "outputs": [],"source": ["qa = QaAgent()\n","engineer = EngineerAgent()"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:40.015305Z",- "start_time": "2024-03-09T00:22:39.951734Z"- }+ "collapsed": false},"id": "79b75d65de738522",- "execution_count": 5+ "execution_count": null},{"cell_type": "markdown",@@ -180,133 +119,51 @@},{"cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': 'You are a software engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.'}], 'model': 'gpt-3.5-turbo', 'temperature': 0.5}}\n",- "DEBUG:httpcore.connection:connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=5.0 socket_options=None\n",- "DEBUG:httpcore.connection:connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x106ce9650>\n",- "DEBUG:httpcore.connection:start_tls.started ssl_context=<ssl.SSLContext object at 0x106c89a30> server_hostname='api.openai.com' timeout=5.0\n",- "DEBUG:httpcore.connection:start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x106cfaf90>\n",- "DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:send_request_headers.complete\n",- "DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:send_request_body.complete\n",- "DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 09 Mar 2024 00:22:40 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-0125'), (b'openai-organization', b'climate-ai-hackathon-13'), (b'openai-processing-ms', b'493'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'5000'), (b'x-ratelimit-limit-tokens', b'160000'), (b'x-ratelimit-remaining-requests', b'4999'), (b'x-ratelimit-remaining-tokens', b'159935'), (b'x-ratelimit-reset-requests', b'12ms'), (b'x-ratelimit-reset-tokens', b'24ms'), (b'x-request-id', b'req_cb817c332291365bd251be9a8d4a1e44'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=JMdInqPp1X2SJyzcq1It6AWAQjWCC9Qj3FVD6sUlBCE-1709943760-1.0.1.1-Ajv2g4sFClAkqugRqVVRwOAIVCuY42uY2XzKD4gs2732_MJ1OeurGvxUVv6QQH2ofussA8WCsOgoRBitrTL14g; path=/; expires=Sat, 09-Mar-24 00:52:40 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=VS0BsYq_1rLiG5u.c2PkBugJVij2Vnia.O05Cywhnm0-1709943760638-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'8616e974596c7aec-SJC'), (b'Content-Encoding', b'br'), (b'alt-svc', b'h3=\":443\"; ma=86400')])\n",- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",- "DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:receive_response_body.complete\n",- "DEBUG:httpcore.http11:response_closed.started\n",- "DEBUG:httpcore.http11:response_closed.complete\n",- "DEBUG:openai._base_client:HTTP Request: POST https://api.openai.com/v1/chat/completions \"200 OK\"\n",- "DEBUG:root:LLM call from agent named: engineer\n",- "ERROR:root:Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are a software engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.'}], 'temperature': 0.5} - skipping upload to AgentOps\n"- ]- }- ],+ "outputs": [],"source": ["generated_func = engineer.completion(\"Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.\")"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:40.712444Z",- "start_time": "2024-03-09T00:22:39.996Z"- }+ "collapsed": false},"id": "69e76061a626549",- "execution_count": 6+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "data": {- "text/plain": "<IPython.core.display.Markdown object>",- "text/markdown": "```python\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n```"- },- "metadata": {},- "output_type": "display_data"- }- ],+ "outputs": [],"source": ["display(Markdown('```python\\n' + generated_func + '\\n```'))"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:40.714212Z",- "start_time": "2024-03-09T00:22:40.713066Z"- }+ "collapsed": false},"id": "830b86dac47dceb3",- "execution_count": 7+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': 'You are a qa engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python unit test that test the following function: \\n def multiply_and_divide(num1, num2):\\n result = (num1 * num2) / 2\\n return result'}], 'model': 'gpt-3.5-turbo', 'temperature': 0.5}}\n",- "DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:send_request_headers.complete\n",- "DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:send_request_body.complete\n",- "DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 09 Mar 2024 00:22:42 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-0125'), (b'openai-organization', b'climate-ai-hackathon-13'), (b'openai-processing-ms', b'1359'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'5000'), (b'x-ratelimit-limit-tokens', b'160000'), (b'x-ratelimit-remaining-requests', b'4999'), (b'x-ratelimit-remaining-tokens', b'159928'), (b'x-ratelimit-reset-requests', b'12ms'), (b'x-ratelimit-reset-tokens', b'27ms'), (b'x-request-id', b'req_d5c37b8eee23eaa70f89ba9b1592babb'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'8616e9789d807aec-SJC'), (b'Content-Encoding', b'br'), (b'alt-svc', b'h3=\":443\"; ma=86400')])\n",- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",- "DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:receive_response_body.complete\n",- "DEBUG:httpcore.http11:response_closed.started\n",- "DEBUG:httpcore.http11:response_closed.complete\n",- "DEBUG:openai._base_client:HTTP Request: POST https://api.openai.com/v1/chat/completions \"200 OK\"\n",- "DEBUG:root:LLM call from agent named: qa\n",- "ERROR:root:Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are a qa engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python unit test that test the following function: \\n def multiply_and_divide(num1, num2):\\n result = (num1 * num2) / 2\\n return result'}], 'temperature': 0.5} - skipping upload to AgentOps\n"- ]- }- ],+ "outputs": [],"source": ["generated_test = qa.completion(\"Write a python unit test that test the following function: \\n \" + generated_func)"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:42.208763Z",- "start_time": "2024-03-09T00:22:40.717809Z"- }+ "collapsed": false},"id": "63c9d0d457aee91a",- "execution_count": 8+ "execution_count": null},{"cell_type": "code",- "outputs": [- {- "data": {- "text/plain": "<IPython.core.display.Markdown object>",- "text/markdown": "```python\nimport unittest\n\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n\nclass TestMultiplyAndDivide(unittest.TestCase):\n\n def test_multiply_and_divide(self):\n self.assertEqual(multiply_and_divide(2, 3), 3)\n self.assertEqual(multiply_and_divide(5, 5), 12.5)\n self.assertEqual(multiply_and_divide(10, 2), 10)\n\nif __name__ == '__main__':\n unittest.main()\n```"- },- "metadata": {},- "output_type": "display_data"- }- ],+ "outputs": [],"source": ["display(Markdown('```python\\n' + generated_test + '\\n```'))"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:42.209552Z",- "start_time": "2024-03-09T00:22:42.207173Z"- }+ "collapsed": false},"id": "a88ffcbd2015d422",- "execution_count": 9+ "execution_count": null},{"cell_type": "markdown",@@ -330,36 +187,7 @@},{"cell_type": "code",- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': 'You are not a tracked agent'}, {'role': 'user', 'content': 'Say hello'}], 'model': 'gpt-3.5-turbo'}}\n",- "DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:send_request_headers.complete\n",- "DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:send_request_body.complete\n",- "DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 09 Mar 2024 00:22:42 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-0125'), (b'openai-organization', b'climate-ai-hackathon-13'), (b'openai-processing-ms', b'368'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'5000'), (b'x-ratelimit-limit-tokens', b'160000'), (b'x-ratelimit-remaining-requests', b'4999'), (b'x-ratelimit-remaining-tokens', b'159973'), (b'x-ratelimit-reset-requests', b'12ms'), (b'x-ratelimit-reset-tokens', b'10ms'), (b'x-request-id', b'req_bf944a1e4f522bb4eee1d91eb505e28b'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'8616e981ef5f7aec-SJC'), (b'Content-Encoding', b'br'), (b'alt-svc', b'h3=\":443\"; ma=86400')])\n",- "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",- "DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>\n",- "DEBUG:httpcore.http11:receive_response_body.complete\n",- "DEBUG:httpcore.http11:response_closed.started\n",- "DEBUG:httpcore.http11:response_closed.complete\n",- "DEBUG:openai._base_client:HTTP Request: POST https://api.openai.com/v1/chat/completions \"200 OK\"\n",- "ERROR:root:Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are not a tracked agent'}, {'role': 'user', 'content': 'Say hello'}]} - skipping upload to AgentOps\n"- ]- },- {- "data": {- "text/plain": "'Hello! How can I assist you today?'"- },- "execution_count": 10,- "metadata": {},- "output_type": "execute_result"- }- ],+ "outputs": [],"source": ["res = openai_client.chat.completions.create(\n"," model=\"gpt-3.5-turbo\",\n",@@ -369,14 +197,10 @@"res.choices[0].message.content"],"metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2024-03-09T00:22:42.787552Z",- "start_time": "2024-03-09T00:22:42.211224Z"- }+ "collapsed": false},"id": "122e923cb07fd5f4",- "execution_count": 10+ "execution_count": null},{"cell_type": "markdown",