Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
supports functions
Braelyn Boynton592c14a04d75
diff --git a/agentops/agent.py b/agentops/agent.pyindex 56e22e63..c2c0b8ca 100644--- a/agentops/agent.py+++ b/agentops/agent.py@@ -1,20 +1,31 @@import uuidfrom agentops import Client+from inspect import isclass, isfunctiondef track_agent(name: str | None):- def class_decorator(cls):- cls._agent_ops_agent_name = name or cls.__name__+ def decorator(obj):+ obj._agent_ops_agent_name = name or obj.__name__- original_init = cls.__init__+ if isclass(obj):+ original_init = obj.__init__- def new_init(self, *args, **kwargs):- self._agent_ops_agent_id = str(uuid.uuid4())+ def new_init(self, *args, **kwargs):+ self._agent_ops_agent_id = str(uuid.uuid4())+ ao_client = Client()+ ao_client.create_agent(self._agent_ops_agent_id, self._agent_ops_agent_name)+ original_init(self, *args, **kwargs)++ obj.__init__ = new_init++ elif isfunction(obj):+ obj._agent_ops_agent_id = str(uuid.uuid4())ao_client = Client()- ao_client.create_agent(self._agent_ops_agent_id, self._agent_ops_agent_name)- original_init(self, *args, **kwargs)+ ao_client.create_agent(obj._agent_ops_agent_id, obj._agent_ops_agent_name)++ else:+ raise Exception("Invalid input, 'obj' must be a class or a function")- cls.__init__ = new_init- return cls+ return obj- return class_decorator+ return decoratordiff --git a/examples/multi_agent_example.ipynb b/examples/multi_agent_example.ipynbindex e1ed0ec0..61638a27 100644--- a/examples/multi_agent_example.ipynb+++ b/examples/multi_agent_example.ipynb@@ -25,10 +25,14 @@"from IPython.display import display, Markdown"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:39.813046Z",+ "start_time": "2024-03-09T00:22:39.580341Z"+ }},"id": "7c566fac57d3b6ce",- "execution_count": null+ "execution_count": 1},{"cell_type": "code",@@ -40,23 +44,51 @@"logging.basicConfig(level=logging.DEBUG) # this will let us see that calls are assigned to an agent"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:39.820790Z",+ "start_time": "2024-03-09T00:22:39.814133Z"+ }},"id": "9f8c52496c04693",- "execution_count": null+ "execution_count": 2},{"cell_type": "code",- "outputs": [],+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): localhost:8000\n",+ "DEBUG:urllib3.connectionpool:http://localhost:8000 \"POST /sessions HTTP/1.1\" 500 35\n",+ "WARNING:root:AgentOps: Could not post data - internal server error\n",+ "INFO:root:View info on this session at https://agentops.ai/dashboard?session_id=749afbc4-2df6-4a7b-bbdb-a2c5b9d745c5\n",+ "DEBUG:httpx:load_ssl_context verify=True cert=None trust_env=True http2=False\n",+ "DEBUG:httpx:load_verify_locations cafile='/opt/homebrew/lib/python3.11/site-packages/certifi/cacert.pem'\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'message': 'Internal Server Error'}\n"+ ]+ }+ ],"source": [- "ao_client = agentops.Client(AGENTOPS_API_KEY)\n",+ "ao_client = agentops.Client(AGENTOPS_API_KEY, endpoint=\"http://localhost:8000\")\n","openai_client = openai.Client()"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:39.945509Z",+ "start_time": "2024-03-09T00:22:39.817178Z"+ }},"id": "af062552554d60ce",- "execution_count": null+ "execution_count": 3},{"cell_type": "markdown",@@ -87,23 +119,52 @@" return res.choices[0].message.content"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:39.951283Z",+ "start_time": "2024-03-09T00:22:39.947705Z"+ }},"id": "727e3cc26ce3ec3",- "execution_count": null+ "execution_count": 4},{"cell_type": "code",- "outputs": [],+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): localhost:8000\n",+ "DEBUG:urllib3.connectionpool:http://localhost:8000 \"POST /agents HTTP/1.1\" 500 35\n",+ "WARNING:root:AgentOps: Could not post data - internal server error\n",+ "DEBUG:urllib3.connectionpool:Starting new HTTP connection (1): localhost:8000\n",+ "DEBUG:urllib3.connectionpool:http://localhost:8000 \"POST /agents HTTP/1.1\" 500 35\n",+ "WARNING:root:AgentOps: Could not post data - internal server error\n"+ ]+ },+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "{'message': 'Internal Server Error'}\n",+ "{'message': 'Internal Server Error'}\n"+ ]+ }+ ],"source": ["qa = QaAgent()\n","engineer = EngineerAgent()"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:40.015305Z",+ "start_time": "2024-03-09T00:22:39.951734Z"+ }},"id": "79b75d65de738522",- "execution_count": null+ "execution_count": 5},{"cell_type": "markdown",@@ -119,51 +180,133 @@},{"cell_type": "code",- "outputs": [],+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': 'You are a software engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.'}], 'model': 'gpt-3.5-turbo', 'temperature': 0.5}}\n",+ "DEBUG:httpcore.connection:connect_tcp.started host='api.openai.com' port=443 local_address=None timeout=5.0 socket_options=None\n",+ "DEBUG:httpcore.connection:connect_tcp.complete return_value=<httpcore._backends.sync.SyncStream object at 0x106ce9650>\n",+ "DEBUG:httpcore.connection:start_tls.started ssl_context=<ssl.SSLContext object at 0x106c89a30> server_hostname='api.openai.com' timeout=5.0\n",+ "DEBUG:httpcore.connection:start_tls.complete return_value=<httpcore._backends.sync.SyncStream object at 0x106cfaf90>\n",+ "DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:send_request_headers.complete\n",+ "DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:send_request_body.complete\n",+ "DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 09 Mar 2024 00:22:40 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-0125'), (b'openai-organization', b'climate-ai-hackathon-13'), (b'openai-processing-ms', b'493'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'5000'), (b'x-ratelimit-limit-tokens', b'160000'), (b'x-ratelimit-remaining-requests', b'4999'), (b'x-ratelimit-remaining-tokens', b'159935'), (b'x-ratelimit-reset-requests', b'12ms'), (b'x-ratelimit-reset-tokens', b'24ms'), (b'x-request-id', b'req_cb817c332291365bd251be9a8d4a1e44'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Set-Cookie', b'__cf_bm=JMdInqPp1X2SJyzcq1It6AWAQjWCC9Qj3FVD6sUlBCE-1709943760-1.0.1.1-Ajv2g4sFClAkqugRqVVRwOAIVCuY42uY2XzKD4gs2732_MJ1OeurGvxUVv6QQH2ofussA8WCsOgoRBitrTL14g; path=/; expires=Sat, 09-Mar-24 00:52:40 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Set-Cookie', b'_cfuvid=VS0BsYq_1rLiG5u.c2PkBugJVij2Vnia.O05Cywhnm0-1709943760638-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None'), (b'Server', b'cloudflare'), (b'CF-RAY', b'8616e974596c7aec-SJC'), (b'Content-Encoding', b'br'), (b'alt-svc', b'h3=\":443\"; ma=86400')])\n",+ "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",+ "DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:receive_response_body.complete\n",+ "DEBUG:httpcore.http11:response_closed.started\n",+ "DEBUG:httpcore.http11:response_closed.complete\n",+ "DEBUG:openai._base_client:HTTP Request: POST https://api.openai.com/v1/chat/completions \"200 OK\"\n",+ "DEBUG:root:LLM call from agent named: engineer\n",+ "ERROR:root:Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are a software engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.'}], 'temperature': 0.5} - skipping upload to AgentOps\n"+ ]+ }+ ],"source": ["generated_func = engineer.completion(\"Write a python function that accepts two numbers and multiplies them together, then divides by two. No example.\")"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:40.712444Z",+ "start_time": "2024-03-09T00:22:39.996Z"+ }},"id": "69e76061a626549",- "execution_count": null+ "execution_count": 6},{"cell_type": "code",- "outputs": [],+ "outputs": [+ {+ "data": {+ "text/plain": "<IPython.core.display.Markdown object>",+ "text/markdown": "```python\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n```"+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],"source": ["display(Markdown('```python\\n' + generated_func + '\\n```'))"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:40.714212Z",+ "start_time": "2024-03-09T00:22:40.713066Z"+ }},"id": "830b86dac47dceb3",- "execution_count": null+ "execution_count": 7},{"cell_type": "code",- "outputs": [],+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': 'You are a qa engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python unit test that test the following function: \\n def multiply_and_divide(num1, num2):\\n result = (num1 * num2) / 2\\n return result'}], 'model': 'gpt-3.5-turbo', 'temperature': 0.5}}\n",+ "DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:send_request_headers.complete\n",+ "DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:send_request_body.complete\n",+ "DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 09 Mar 2024 00:22:42 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-0125'), (b'openai-organization', b'climate-ai-hackathon-13'), (b'openai-processing-ms', b'1359'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'5000'), (b'x-ratelimit-limit-tokens', b'160000'), (b'x-ratelimit-remaining-requests', b'4999'), (b'x-ratelimit-remaining-tokens', b'159928'), (b'x-ratelimit-reset-requests', b'12ms'), (b'x-ratelimit-reset-tokens', b'27ms'), (b'x-request-id', b'req_d5c37b8eee23eaa70f89ba9b1592babb'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'8616e9789d807aec-SJC'), (b'Content-Encoding', b'br'), (b'alt-svc', b'h3=\":443\"; ma=86400')])\n",+ "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",+ "DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:receive_response_body.complete\n",+ "DEBUG:httpcore.http11:response_closed.started\n",+ "DEBUG:httpcore.http11:response_closed.complete\n",+ "DEBUG:openai._base_client:HTTP Request: POST https://api.openai.com/v1/chat/completions \"200 OK\"\n",+ "DEBUG:root:LLM call from agent named: qa\n",+ "ERROR:root:Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are a qa engineer and only output python code, no markdown tags.'}, {'role': 'user', 'content': 'Write a python unit test that test the following function: \\n def multiply_and_divide(num1, num2):\\n result = (num1 * num2) / 2\\n return result'}], 'temperature': 0.5} - skipping upload to AgentOps\n"+ ]+ }+ ],"source": ["generated_test = qa.completion(\"Write a python unit test that test the following function: \\n \" + generated_func)"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:42.208763Z",+ "start_time": "2024-03-09T00:22:40.717809Z"+ }},"id": "63c9d0d457aee91a",- "execution_count": null+ "execution_count": 8},{"cell_type": "code",- "outputs": [],+ "outputs": [+ {+ "data": {+ "text/plain": "<IPython.core.display.Markdown object>",+ "text/markdown": "```python\nimport unittest\n\ndef multiply_and_divide(num1, num2):\n result = (num1 * num2) / 2\n return result\n\nclass TestMultiplyAndDivide(unittest.TestCase):\n\n def test_multiply_and_divide(self):\n self.assertEqual(multiply_and_divide(2, 3), 3)\n self.assertEqual(multiply_and_divide(5, 5), 12.5)\n self.assertEqual(multiply_and_divide(10, 2), 10)\n\nif __name__ == '__main__':\n unittest.main()\n```"+ },+ "metadata": {},+ "output_type": "display_data"+ }+ ],"source": ["display(Markdown('```python\\n' + generated_test + '\\n```'))"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:42.209552Z",+ "start_time": "2024-03-09T00:22:42.207173Z"+ }},"id": "a88ffcbd2015d422",- "execution_count": null+ "execution_count": 9},{"cell_type": "markdown",@@ -187,7 +330,36 @@},{"cell_type": "code",- "outputs": [],+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "DEBUG:openai._base_client:Request options: {'method': 'post', 'url': '/chat/completions', 'files': None, 'json_data': {'messages': [{'role': 'system', 'content': 'You are not a tracked agent'}, {'role': 'user', 'content': 'Say hello'}], 'model': 'gpt-3.5-turbo'}}\n",+ "DEBUG:httpcore.http11:send_request_headers.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:send_request_headers.complete\n",+ "DEBUG:httpcore.http11:send_request_body.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:send_request_body.complete\n",+ "DEBUG:httpcore.http11:receive_response_headers.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:receive_response_headers.complete return_value=(b'HTTP/1.1', 200, b'OK', [(b'Date', b'Sat, 09 Mar 2024 00:22:42 GMT'), (b'Content-Type', b'application/json'), (b'Transfer-Encoding', b'chunked'), (b'Connection', b'keep-alive'), (b'access-control-allow-origin', b'*'), (b'Cache-Control', b'no-cache, must-revalidate'), (b'openai-model', b'gpt-3.5-turbo-0125'), (b'openai-organization', b'climate-ai-hackathon-13'), (b'openai-processing-ms', b'368'), (b'openai-version', b'2020-10-01'), (b'strict-transport-security', b'max-age=15724800; includeSubDomains'), (b'x-ratelimit-limit-requests', b'5000'), (b'x-ratelimit-limit-tokens', b'160000'), (b'x-ratelimit-remaining-requests', b'4999'), (b'x-ratelimit-remaining-tokens', b'159973'), (b'x-ratelimit-reset-requests', b'12ms'), (b'x-ratelimit-reset-tokens', b'10ms'), (b'x-request-id', b'req_bf944a1e4f522bb4eee1d91eb505e28b'), (b'CF-Cache-Status', b'DYNAMIC'), (b'Server', b'cloudflare'), (b'CF-RAY', b'8616e981ef5f7aec-SJC'), (b'Content-Encoding', b'br'), (b'alt-svc', b'h3=\":443\"; ma=86400')])\n",+ "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",+ "DEBUG:httpcore.http11:receive_response_body.started request=<Request [b'POST']>\n",+ "DEBUG:httpcore.http11:receive_response_body.complete\n",+ "DEBUG:httpcore.http11:response_closed.started\n",+ "DEBUG:httpcore.http11:response_closed.complete\n",+ "DEBUG:openai._base_client:HTTP Request: POST https://api.openai.com/v1/chat/completions \"200 OK\"\n",+ "ERROR:root:Unable to parse a chunk for LLM call {'model': 'gpt-3.5-turbo', 'messages': [{'role': 'system', 'content': 'You are not a tracked agent'}, {'role': 'user', 'content': 'Say hello'}]} - skipping upload to AgentOps\n"+ ]+ },+ {+ "data": {+ "text/plain": "'Hello! How can I assist you today?'"+ },+ "execution_count": 10,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["res = openai_client.chat.completions.create(\n"," model=\"gpt-3.5-turbo\",\n",@@ -197,10 +369,14 @@"res.choices[0].message.content"],"metadata": {- "collapsed": false+ "collapsed": false,+ "ExecuteTime": {+ "end_time": "2024-03-09T00:22:42.787552Z",+ "start_time": "2024-03-09T00:22:42.211224Z"+ }},"id": "122e923cb07fd5f4",- "execution_count": null+ "execution_count": 10},{"cell_type": "markdown",