Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
record_function now using singleton and in its own file decorators.py
Howard Gil5733d07af832
diff --git a/README.md b/README.mdindex 2bdce65f..d90d1da5 100644--- a/README.md+++ b/README.md@@ -38,7 +38,7 @@ ao_client = agentops.Client(<INSERT YOUR API KEY HERE>)...# (optional: record specific functions)-@ao_client.record_action('sample function being record')+@record_function('sample function being record')def sample_function(...):...diff --git a/agentops/__init__.py b/agentops/__init__.pyindex f560821b..d5ab55d1 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -4,3 +4,4 @@ from .client import Clientfrom .event import ActionEvent, LLMEvent, ToolEvent, ErrorEventfrom .logger import AgentOpsLoggerfrom .enums import Models+from .decorators import record_functiondiff --git a/agentops/client.py b/agentops/client.pyindex 6933629c..762f1505 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -5,7 +5,7 @@ Classes:Client: Provides methods to interact with the AgentOps service."""-from .event import Event, ActionEvent, ErrorEvent+from .event import ActionEvent, ErrorEvent, Eventfrom .helpers import get_ISO_time, singleton, check_call_stack_for_agent_idfrom .session import Sessionfrom .worker import Worker@@ -14,7 +14,6 @@ from uuid import uuid4from typing import Optional, Listfrom pydantic import Fieldfrom os import environ-import functoolsimport tracebackimport loggingimport inspect@@ -110,33 +109,6 @@ class Client(metaclass=MetaClient):logging.warning("AgentOps: Cannot record event - no current session")- def record_function(self, event_name: str):- """- Decorator to record an event before and after a function call.- Usage:- - Actions: Records function parameters and return statements of the- function being decorated. Additionally, timing information about- the action is recorded- Args:- event_name (str): The name of the event to record.- """-- def decorator(func):- if inspect.iscoroutinefunction(func):- @functools.wraps(func)- async def async_wrapper(*args, **kwargs):- return await self._record_event_async(func, event_name, *args, **kwargs)-- return async_wrapper- else:- @functools.wraps(func)- def sync_wrapper(*args, **kwargs):- return self._record_event_sync(func, event_name, *args, **kwargs)-- return sync_wrapper-- return decorator-def _record_event_sync(self, func, event_name, *args, **kwargs):init_time = get_ISO_time()func_args = inspect.signature(func).parametersdiff --git a/agentops/decorators.py b/agentops/decorators.pynew file mode 100644index 00000000..165512ac--- /dev/null+++ b/agentops/decorators.py@@ -0,0 +1,35 @@+from .client import Client+from .event import Event+import inspect+import functools+++def record_function(event_name: str):+ """+ Decorator to record an event before and after a function call.+ Usage:+ - Actions: Records function parameters and return statements of the+ function being decorated. Additionally, timing information about+ the action is recorded+ Args:+ event_name (str): The name of the event to record.+ """++ ao_client = Client()++ def decorator(func):++ if inspect.iscoroutinefunction(func):+ @functools.wraps(func)+ async def async_wrapper(*args, **kwargs):+ return await ao_client._record_event_async(func, event_name, *args, **kwargs)++ return async_wrapper+ else:+ @functools.wraps(func)+ def sync_wrapper(*args, **kwargs):+ return ao_client._record_event_sync(func, event_name, *args, **kwargs)++ return sync_wrapper++ return decoratordiff --git a/examples/openai-gpt.ipynb b/examples/openai-gpt.ipynbindex 2a686332..1d8615d0 100644--- a/examples/openai-gpt.ipynb+++ b/examples/openai-gpt.ipynb@@ -2,28 +2,36 @@"cells": [{"cell_type": "markdown",- "source": [- "# Monitoring \n",- "This is an example of how to use the AgentOps library for basic Agent monitoring with OpenAI's GPT"- ],+ "id": "212799f5255f3513","metadata": {"collapsed": false},- "id": "212799f5255f3513"+ "source": [+ "# Monitoring \n",+ "This is an example of how to use the AgentOps library for basic Agent monitoring with OpenAI's GPT"+ ]},{"cell_type": "markdown",- "source": [- "_At this time, make sure to install openai==0.28.1_"- ],+ "id": "6d717e4c8bf0de9c","metadata": {"collapsed": false},- "id": "6d717e4c8bf0de9c"+ "source": [+ "_At this time, make sure to install openai==0.28.1_"+ ]},{"cell_type": "code",- "execution_count": 3,+ "execution_count": null,+ "id": "c51dcbda80eb8c53",+ "metadata": {+ "ExecuteTime": {+ "end_time": "2023-12-15T21:33:47.472626Z",+ "start_time": "2023-12-15T21:33:47.448868Z"+ },+ "collapsed": false+ },"outputs": [],"source": ["import openai\n",@@ -31,169 +39,160 @@"from dotenv import load_dotenv\n","import os\n","from openai import ChatCompletion"- ],- "metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2023-12-15T21:33:47.472626Z",- "start_time": "2023-12-15T21:33:47.448868Z"- }- },- "id": "c51dcbda80eb8c53"+ ]},{"cell_type": "markdown",- "source": [- "Next, we'll grab our two API keys. You can use dotenv like below or however else you like to load environment variables"- ],+ "id": "33bb35b82e9a8f71","metadata": {"collapsed": false},- "id": "33bb35b82e9a8f71"+ "source": [+ "Next, we'll grab our two API keys. You can use dotenv like below or however else you like to load environment variables"+ ]},{"cell_type": "code",- "execution_count": 4,- "outputs": [],- "source": [- "load_dotenv()\n",- "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY') or \"<your_openai_key>\"\n",- "AGENTOPS_API_KEY = os.getenv('AGENTOPS_API_KEY') or \"<your_agentops_key>\""- ],+ "execution_count": null,+ "id": "cbc5d13679e269dc","metadata": {- "collapsed": false,"ExecuteTime": {"end_time": "2023-12-15T21:33:47.473321Z","start_time": "2023-12-15T21:33:47.460144Z"- }+ },+ "collapsed": false},- "id": "cbc5d13679e269dc"+ "outputs": [],+ "source": [+ "load_dotenv()\n",+ "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY') or \"<your_openai_key>\"\n",+ "AGENTOPS_API_KEY = os.getenv('AGENTOPS_API_KEY') or \"<your_agentops_key>\""+ ]},{"cell_type": "markdown",- "source": [- "The AgentOps library is designed to be a plug-and-play replacement for the OpenAI Client, maximizing use with minimal install effort."- ],+ "id": "82fdff21ff00ef37","metadata": {"collapsed": false},- "id": "82fdff21ff00ef37"+ "source": [+ "The AgentOps library is designed to be a plug-and-play replacement for the OpenAI Client, maximizing use with minimal install effort."+ ]},{"cell_type": "code",- "execution_count": 5,- "outputs": [],- "source": [- "ao_client = agentops.Client(AGENTOPS_API_KEY)\n",- "openai.api_key = OPENAI_API_KEY"- ],+ "execution_count": null,+ "id": "fe8116d5969f1d23","metadata": {- "collapsed": false,"ExecuteTime": {"end_time": "2023-12-15T21:33:48.179508Z","start_time": "2023-12-15T21:33:47.463468Z"- }+ },+ "collapsed": false},- "id": "fe8116d5969f1d23"+ "outputs": [],+ "source": [+ "ao_client = agentops.Client(AGENTOPS_API_KEY)\n",+ "openai.api_key = OPENAI_API_KEY"+ ]},{"cell_type": "markdown",- "source": [- "Now just use OpenAI as you would normally!"- ],+ "id": "3c20bbfa91b3419c","metadata": {"collapsed": false},- "id": "3c20bbfa91b3419c"+ "source": [+ "Now just use OpenAI as you would normally!"+ ]},{"cell_type": "markdown",- "source": [- "## Single Session with ChatCompletion"- ],+ "id": "b42f5685ac4af5c2","metadata": {"collapsed": false},- "id": "b42f5685ac4af5c2"+ "source": [+ "## Single Session with ChatCompletion"+ ]},{"cell_type": "code",- "execution_count": 6,- "outputs": [- {- "data": {- "text/plain": "<OpenAIObject chat.completion id=chatcmpl-8WA6WNna1UiKr7dg1F1fGtAuTRZeB at 0x116b48860> JSON: {\n \"id\": \"chatcmpl-8WA6WNna1UiKr7dg1F1fGtAuTRZeB\",\n \"object\": \"chat.completion\",\n \"created\": 1702676028,\n \"model\": \"gpt-3.5-turbo-0613\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"Hello! How can I assist you today?\"\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 8,\n \"completion_tokens\": 9,\n \"total_tokens\": 17\n },\n \"system_fingerprint\": null\n}"- },- "execution_count": 6,- "metadata": {},- "output_type": "execute_result"- }- ],- "source": [- "message = {\"role\": \"user\", \"content\": \"Hello\"},\n",- "ChatCompletion.create(\n",- " model='gpt-3.5-turbo', messages=message, temperature=0.5)"- ],+ "execution_count": null,+ "id": "9cd47d3fa1e252e1","metadata": {- "collapsed": false,"ExecuteTime": {"end_time": "2023-12-15T21:33:49.022014Z","start_time": "2023-12-15T21:33:48.181090Z"- }+ },+ "collapsed": false},- "id": "9cd47d3fa1e252e1"+ "outputs": [],+ "source": [+ "message = {\"role\": \"user\", \"content\": \"Hello\"},\n",+ "ChatCompletion.create(\n",+ " model='gpt-3.5-turbo', messages=message, temperature=0.5)"+ ]},{"cell_type": "markdown",- "source": [- "Make sure to end your session with a `Result` (Success|Fail|Indeterminate) for better tracking"- ],+ "id": "bf75276ad9fbb3f4","metadata": {"collapsed": false},- "id": "bf75276ad9fbb3f4"+ "source": [+ "Make sure to end your session with a `Result` (Success|Fail|Indeterminate) for better tracking"+ ]},{"cell_type": "code",- "execution_count": 7,- "outputs": [],- "source": [- "ao_client.end_session('Success')"- ],+ "execution_count": null,+ "id": "f59fe80a7e00e6e8","metadata": {- "collapsed": false,"ExecuteTime": {"end_time": "2023-12-15T21:33:50.619934Z","start_time": "2023-12-15T21:33:49.021089Z"- }+ },+ "collapsed": false},- "id": "f59fe80a7e00e6e8"+ "outputs": [],+ "source": [+ "ao_client.end_session('Success')"+ ]},{"cell_type": "markdown",- "source": [- "Now if you check the AgentOps dashboard, you should see information related to this run!"- ],+ "id": "318a7186c1be2d59","metadata": {"collapsed": false},- "id": "318a7186c1be2d59"+ "source": [+ "Now if you check the AgentOps dashboard, you should see information related to this run!"+ ]},{"cell_type": "markdown",+ "id": "ccf998561cb9a834",+ "metadata": {+ "collapsed": false+ },"source": ["# Events\n","Additionally, you can track custom events via AgentOps.\n","Lets record an event when an LLM says \"Hello\""- ],- "metadata": {- "collapsed": false- },- "id": "ccf998561cb9a834"+ ]},{"cell_type": "code",- "execution_count": 8,+ "execution_count": null,+ "id": "f5a1a63ff4ecf127",+ "metadata": {+ "ExecuteTime": {+ "end_time": "2023-12-15T21:33:50.914863Z",+ "start_time": "2023-12-15T21:33:50.620442Z"+ },+ "collapsed": false+ },"outputs": [],"source": ["# Create new session\n",@@ -201,77 +200,61 @@"\n","# Optionally, we can add tags to the session\n","# ao_client.start_session(['Hello Tracker'])"- ],- "metadata": {- "collapsed": false,- "ExecuteTime": {- "end_time": "2023-12-15T21:33:50.914863Z",- "start_time": "2023-12-15T21:33:50.620442Z"- }- },- "id": "f5a1a63ff4ecf127"+ ]},{"cell_type": "code",- "execution_count": 9,- "outputs": [],- "source": [- "message = {\"role\": \"user\", \"content\": \"Hello\"},\n",- "response = ChatCompletion.create(\n",- " model='gpt-3.5-turbo', messages=message, temperature=0.5)"- ],+ "execution_count": null,+ "id": "7daafba014c1a4d8","metadata": {- "collapsed": false,"ExecuteTime": {"end_time": "2023-12-15T21:33:51.321518Z","start_time": "2023-12-15T21:33:50.915612Z"- }+ },+ "collapsed": false},- "id": "7daafba014c1a4d8"+ "outputs": [],+ "source": [+ "message = {\"role\": \"user\", \"content\": \"Hello\"},\n",+ "response = ChatCompletion.create(\n",+ " model='gpt-3.5-turbo', messages=message, temperature=0.5)"+ ]},{"cell_type": "code",- "execution_count": 12,- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "📝 Recording Event\n"- ]- }- ],- "source": [- "from agentops import Event\n",- "\n",- "if \"hello\" in str(response.choices[0].message.content).lower():\n",- " print('📝 Recording Event')\n",- " ao_client.record(Event(event_type=\"said_hello\"))"- ],+ "execution_count": null,+ "id": "b45754a57148eed1","metadata": {- "collapsed": false,"ExecuteTime": {"end_time": "2023-12-15T21:34:24.301681Z","start_time": "2023-12-15T21:34:24.297258Z"- }+ },+ "collapsed": false},- "id": "b45754a57148eed1"+ "outputs": [],+ "source": [+ "from agentops import ActionEvent\n",+ "\n",+ "if \"hello\" in str(response.choices[0].message.content).lower():\n",+ " print('📝 Recording Event')\n",+ " ao_client.record(ActionEvent(action_type=\"said hello\"))"+ ]},{"cell_type": "code",- "execution_count": 13,- "outputs": [],- "source": [- "ao_client.end_session('Success')"- ],+ "execution_count": null,+ "id": "4ca2b49fc06adddb","metadata": {- "collapsed": false,"ExecuteTime": {"end_time": "2023-12-15T21:34:26.439101Z","start_time": "2023-12-15T21:34:26.436252Z"- }+ },+ "collapsed": false},- "id": "4ca2b49fc06adddb"+ "outputs": [],+ "source": [+ "ao_client.end_session('Success')"+ ]}],"metadata": {diff --git a/tests/openai_handlers/test_llm_tracker_ge_1_async.py b/tests/openai_handlers/test_llm_tracker_ge_1_async.pyindex e4f445c6..a216445c 100644--- a/tests/openai_handlers/test_llm_tracker_ge_1_async.py+++ b/tests/openai_handlers/test_llm_tracker_ge_1_async.py@@ -1,14 +1,12 @@from openai import AsyncOpenAI # Assuming AsyncOpenAI is the async version of the clientimport asyncioimport agentops-from packaging.version import parse-from importlib import import_module-import sys+from agentops import record_functionao_client = agentops.Client()-@ao_client.record_function('sample function being recorded')+@record_function('sample function being recorded')async def call_openai_async():client = AsyncOpenAI() # Using the async clientdiff --git a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/test_llm_tracker_ge_1_sync.pyindex f4a17ce4..7fcfba19 100644--- a/tests/openai_handlers/test_llm_tracker_ge_1_sync.py+++ b/tests/openai_handlers/test_llm_tracker_ge_1_sync.py@@ -1,5 +1,6 @@from openai import OpenAIimport agentops+from agentops import record_functionfrom packaging.version import parsefrom importlib import import_moduleimport sys@@ -16,7 +17,7 @@ if api in sys.modules:print('openai version: ', module_version)-@ao_client.record_function('sample function being recorded')+@record_function('sample function being recorded')def call_openai():client = OpenAI()diff --git a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/test_llm_tracker_lt_1_sync.pyindex 10297ad4..70f917bf 100644--- a/tests/openai_handlers/test_llm_tracker_lt_1_sync.py+++ b/tests/openai_handlers/test_llm_tracker_lt_1_sync.py@@ -1,11 +1,11 @@import openaifrom openai import ChatCompletionimport agentops+from agentops import record_functionfrom packaging.version import parsefrom importlib import import_moduleimport sys-ao_client = agentops.Client()api = 'openai'@@ -17,7 +17,7 @@ if api in sys.modules:print('openai version: ', module_version)-@ao_client.record_function('sample function being recorded')+@record_function('sample function being recorded')def call_openai():message = [{"role": "user", "content": "Hello"},diff --git a/tests/test_session.py b/tests/test_session.pyindex 8a0e37d4..128b9c49 100644--- a/tests/test_session.py+++ b/tests/test_session.py@@ -4,7 +4,7 @@ import timefrom datetime import datetime-from agentops import Client, Event+from agentops import Client, Event, record_function@pytest.fixture@@ -94,7 +94,7 @@ class TestRecordAction:self.client.end_session(end_state='Success')def test_record_function_decorator(self, mock_req):- @self.client.record_function(event_name=self.event_type)+ @record_function(event_name=self.event_type)def add_two(x, y):return x + y@@ -114,7 +114,7 @@ class TestRecordAction:def test_record_function_decorator_multiple(self, mock_req):# Arrange- @self.client.record_function(event_name=self.event_type)+ @record_function(event_name=self.event_type)def add_three(x, y, z=3):return x + y + z@@ -135,7 +135,7 @@ class TestRecordAction:@pytest.mark.asyncioasync def test_async_function_call(self, mock_req):- @self.client.record_function(self.event_type)+ @record_function(self.event_type)async def async_add(x, y):time.sleep(0.1)return x + y@@ -165,7 +165,7 @@ class TestRecordAction:# Arrangeprompt = 'prompt'- @self.client.record_function(event_name=self.event_type)+ @record_function(event_name=self.event_type)def foo(prompt=prompt):return 'output'@@ -185,7 +185,7 @@ class TestRecordAction:# Arrangeprompt = 'prompt'- @self.client.record_function(event_name=self.event_type)+ @record_function(event_name=self.event_type)def llm_call(prompt=prompt):return 'output'