Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
llm tracker (#31)
Shawn Qiu05ddef5cc234
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 99bdd366..4dc51d37 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -1,4 +1,4 @@from .client import Clientfrom .event import Eventfrom .logger import AgentOpsLogger-from .helpers import Models, ActionType+from .helpers import Modelsdiff --git a/agentops/client.py b/agentops/client.pyindex cd16284b..af60353d 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -6,7 +6,7 @@ Classes:"""from .event import Event-from .helpers import Models, ActionType, get_ISO_time+from .helpers import Models, get_ISO_timefrom .session import Sessionfrom .worker import Workerfrom uuid import uuid4@@ -20,6 +20,7 @@ import signalimport sysfrom .config import Configuration+from .llm_tracker import LlmTrackerclass Client:@@ -58,6 +59,10 @@ class Client:self.start_session(tags)+ if 'openai' in sys.modules:+ self.llm_tracker = LlmTracker(self)+ self.llm_tracker.override_api('openai')+def handle_exception(self, exc_type, exc_value, exc_traceback):"""Handle uncaught exceptions before they result in program termination.@@ -101,7 +106,9 @@ class Client:" Start a new session to record again.")def record_action(self, event_name: str,- action_type: ActionType = ActionType.ACTION,+ action_type: str = Field("action",+ description="Type of action that the user is recording",+ pattern="^(action|api|llm)$"),model: Optional[Models] = None,tags: Optional[List[str]] = None):"""@@ -138,7 +145,7 @@ class Client:TOOD: Currently not implemented, coming soon.Args:event_name (str): The name of the event to record.- action_type (ActionType, optional): The type of the event being recorded.+ action_type (str, optional): The type of the event being recorded.Events default to 'action'. Other options include 'api' and 'llm'.model (Models, optional): The model used during the event if an LLM is used (i.e. GPT-4).For models, see the types available in the Models enum.@@ -168,15 +175,15 @@ class Client:# calculating priceaction = action_typeif bool(model):- action = ActionType.LLM+ action = 'llm'if not bool(prompt):raise ValueError("Prompt is required when model is provided.")# Throw error if action type is 'llm' but no model is specified- if action == ActionType.LLM and not bool(model):+ if action == 'llm' and not bool(model):raise ValueError(- f"`model` is a required parameter if `action_type` is set as {ActionType.LLM}. " ++ f"`model` is a required parameter if `action_type` is set as `llm`. " +f"Model can be set as: {list([mod.value for mod in Models])}")try:diff --git a/agentops/event.py b/agentops/event.pyindex e069df91..1bc0b2ab 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -4,7 +4,7 @@ AgentOps events.Classes:Event: Represents discrete events to be recorded."""-from .helpers import get_ISO_time, ActionType, Models+from .helpers import get_ISO_time, Modelsfrom typing import Optional, Listfrom pydantic import Field@@ -18,7 +18,7 @@ class Event:params (str, optional): The parameters passed to the operation.returns (str, optional): The output of the operation.result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate". Defaults to "Indeterminate".- action_type (ActionType, optional): Type of action of the event e.g. 'action', 'llm', 'api'. Defaults to ActionType.ACTION.+ action_type (str, optional): Type of action of the event e.g. 'action', 'llm', 'api'. Defaults to 'action'.model (Models, optional): The model used during the event if an LLM is used (i.e. GPT-4).For models, see the types available in the Models enum.If a model is set but an action_type is not, the action_type will be coerced to 'llm'.@@ -33,7 +33,7 @@ class Event:params (str, optional): The parameters passed to the operation.returns (str, optional): The output of the operation.result (str): Result of the operation.- action_type (ActionType): Type of action of the event.+ action_type (str): Type of action of the event.model (Models, optional): The model used during the event.prompt (str, optional): The input prompt for an LLM call.tags (List[str], optional): Tags associated with the event.@@ -47,7 +47,9 @@ class Event:result: str = Field("Indeterminate",description="Result of the operation",pattern="^(Success|Fail|Indeterminate)$"),- action_type: Optional[ActionType] = ActionType.ACTION,+ action_type: Optional[str] = Field("action",+ description="Type of action that the user is recording",+ pattern="^(action|api|llm)$"),model: Optional[Models] = None,prompt: Optional[str] = None,tags: Optional[List[str]] = None,diff --git a/agentops/helpers.py b/agentops/helpers.pyindex e77d983b..25859cbc 100644--- a/agentops/helpers.py+++ b/agentops/helpers.py@@ -19,9 +19,3 @@ class Models(Enum):GPT_4_32K_0314 = "gpt-4-32k-0314"GPT_4_0613 = "gpt-4-0613"TEXT_EMBEDDING_ADA_002 = "text-embedding-ada-002"---class ActionType:- LLM = "llm"- API = "api"- ACTION = "action"diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pynew file mode 100644index 00000000..a4b3c535--- /dev/null+++ b/agentops/llm_tracker.py@@ -0,0 +1,151 @@+import asyncio+import functools+import inspect+from importlib import import_module+from .event import Event+from .helpers import get_ISO_time+++class LlmTracker:+ SUPPORTED_APIS = {+ 'openai': (+ "Edit.create",+ "Completion.create",+ "ChatCompletion.create",+ "Edit.acreate",+ "Completion.acreate",+ "ChatCompletion.acreate",+ )+ }++ def __init__(self, client):+ self.client = client+ self.event_stream = None++ def parse_and_record_event(self, api, result, kwargs, init_timestamp):+ if api == 'openai':+ event = Event(+ event_type=result.get('object'),+ params=kwargs,+ result='Success',+ returns={"content":+ result['choices'][0]['message']['content']},+ action_type='llm',+ model=result['model'],+ prompt=kwargs['messages'],+ init_timestamp=init_timestamp+ )+ self.client.record(event)++ async def parse_and_record_async(self, api, result, kwargs, init_timestamp):+ if api == 'openai':+ event = Event(+ event_type=result.get('object'),+ params=kwargs,+ result='Success',+ returns={"content":+ result['choices'][0]['message']['content']},+ action_type='llm',+ model=result['model'],+ prompt=kwargs['messages'],+ init_timestamp=init_timestamp+ )+ self.client.record(event)++ def parse_and_record_chunks(self, api, result, kwargs, init_timestamp):+ if api == 'openai':+ model = result.get('model')+ choices = result['choices']+ token = choices[0]['delta'].get('content', '')+ finish_reason = choices[0]['finish_reason']++ if self.event_stream == None:+ self.event_stream = Event(+ event_type='openai stream',+ params=kwargs,+ result='Success',+ returns={"finish_reason": None, "content": token},+ action_type='llm',+ model=model,+ prompt='test prompt',+ init_timestamp=init_timestamp+ )+ else:+ self.event_stream.returns['content'] += token++ # Finish reason is 'stop' or something else+ if bool(finish_reason):+ self.event_stream.returns['finish_reason'] = finish_reason+ self.client.record(self.event_stream)+ self.event_stream = None++ def _override_method(self, api, original_method):+ """+ Generate a new method (either async or sync) that overrides the original+ and records an event when called.+ """++ if inspect.iscoroutinefunction(original_method):+ # Handle async generator for streams+ @functools.wraps(original_method)+ async def async_method(*args, **kwargs):+ init_timestamp = get_ISO_time()+ async_result = await original_method(*args, **kwargs)+ # Async non-stream+ try:+ await self.parse_and_record_async(api, async_result,+ kwargs, init_timestamp)+ return async_result+ # Async stream+ except:+ async def generator():+ async for result in async_result:+ self.parse_and_record_chunks(+ api, result, kwargs, init_timestamp)+ yield result++ return generator()+ return async_method++ # Handle sync code+ else:+ @functools.wraps(original_method)+ def sync_method(*args, **kwargs):+ init_timestamp = get_ISO_time()+ result = original_method(*args, **kwargs)+ # Sync stream+ try:+ self.parse_and_record_event(+ api, result, kwargs, init_timestamp)+ return result+ # Sync non-stream+ except:+ def generator():+ for res in result:+ self.parse_and_record_chunks(+ api, res, kwargs, init_timestamp)+ yield res++ return generator()++ return sync_method++ def override_api(self, api):+ """+ Overrides key methods of the specified API to record events.+ """+ if api not in self.SUPPORTED_APIS:+ raise ValueError(f"Unsupported API: {api}")++ module = import_module(api)++ for method_path in self.SUPPORTED_APIS[api]:+ method_parts = method_path.split(".")+ original_method = functools.reduce(getattr, method_parts, module)+ new_method = self._override_method(api, original_method)++ if len(method_parts) == 1:+ setattr(module, method_parts[0], new_method)+ else:+ parent = functools.reduce(getattr, method_parts[:-1], module)+ setattr(parent, method_parts[-1], new_method)diff --git a/setup.py b/setup.pyindex 0edceeb8..d26181ee 100644--- a/setup.py+++ b/setup.py@@ -7,7 +7,7 @@ setup(author_email='[email protected]',packages=find_packages(),install_requires=[- 'requests'+ 'requests', 'pydantic'],extras_require={'dev': [diff --git a/tests/api_call_agent.py b/tests/api_call_agent.pynew file mode 100644index 00000000..6f5397b7--- /dev/null+++ b/tests/api_call_agent.py@@ -0,0 +1,18 @@+import openai+from openai import ChatCompletion++import agentops++ao_client = agentops.Client(api_key='',+ endpoint="http://localhost:8000",+ tags=['mock tests'])++openai.api_key = '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'++message = [{"role": "user", "content": "Hello"},+ {"role": "assistant", "content": "Hi there!"}]++ChatCompletion.create(+ model='gpt-3.5-turbo', messages=message, temperature=0.5)++ao_client.end_session('Success')diff --git a/tests/async_call_agent.py b/tests/async_call_agent.pynew file mode 100644index 00000000..4195e529--- /dev/null+++ b/tests/async_call_agent.py@@ -0,0 +1,66 @@+import agentops+import asyncio+import openai+++# from wandb.integration.openai import autolog+# autolog()+++ao_client = agentops.Client(api_key='',+ tags=['mock tests'])+++openai.api_key = '3'+++async def stream_achat():+ message = [{"role": "user", "content": "stream achat test"}]++ res = await openai.ChatCompletion.acreate(+ model='gpt-3.5-turbo', messages=message, temperature=0.5, stream=True)+ async for r in res:+ print(r)+ print('*'*100)+++def stream_chat():+ message = [{"role": "user", "content": "stream chat test"}]+ res = openai.ChatCompletion.create(+ model='gpt-3.5-turbo', messages=message, temperature=0.5, stream=True)+ for r in res:+ print(r)+ print('*'*100)+++async def achat():+ message = [{"role": "user", "content": "achat test"}]+ res = await openai.ChatCompletion.acreate(+ model='gpt-3.5-turbo', messages=message, temperature=0.5)+ print(res)++ print('*'*100)+++def chat():+ message = [{"role": "user", "content": "chat test"}]+ res = openai.ChatCompletion.create(+ model='gpt-3.5-turbo', messages=message, temperature=0.5)+ print(res)++ print('*'*100)+++print('running async stream')+asyncio.run(stream_achat())++print('running sync stream')+stream_chat()++print('running async chat')+asyncio.run(achat())++print('running sync chat')+chat()++ao_client.end_session('Success')diff --git a/tox.ini b/tox.iniindex 37402e4f..25fac5bf 100644--- a/tox.ini+++ b/tox.ini@@ -11,7 +11,9 @@ deps =pytestrequests_mockcoverage+ pydanticmypy: mypy+ types-requestscommands =coverage run --source agentops -m pytestcoverage report -m