Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

llm tracker (#31)

Shawn Qiu05ddef5cc234
diff --git a/agentops/__init__.py b/agentops/__init__.py
index 99bdd366..4dc51d37 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -1,4 +1,4 @@
from .client import Client
from .event import Event
from .logger import AgentOpsLogger
-from .helpers import Models, ActionType
+from .helpers import Models
diff --git a/agentops/client.py b/agentops/client.py
index cd16284b..af60353d 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -6,7 +6,7 @@ Classes:
"""
from .event import Event
-from .helpers import Models, ActionType, get_ISO_time
+from .helpers import Models, get_ISO_time
from .session import Session
from .worker import Worker
from uuid import uuid4
@@ -20,6 +20,7 @@ import signal
import sys
from .config import Configuration
+from .llm_tracker import LlmTracker
class Client:
@@ -58,6 +59,10 @@ class Client:
self.start_session(tags)
+ if 'openai' in sys.modules:
+ self.llm_tracker = LlmTracker(self)
+ self.llm_tracker.override_api('openai')
+
def handle_exception(self, exc_type, exc_value, exc_traceback):
"""
Handle uncaught exceptions before they result in program termination.
@@ -101,7 +106,9 @@ class Client:
" Start a new session to record again.")
def record_action(self, event_name: str,
- action_type: ActionType = ActionType.ACTION,
+ action_type: str = Field("action",
+ description="Type of action that the user is recording",
+ pattern="^(action|api|llm)$"),
model: Optional[Models] = None,
tags: Optional[List[str]] = None):
"""
@@ -138,7 +145,7 @@ class Client:
TOOD: Currently not implemented, coming soon.
Args:
event_name (str): The name of the event to record.
- action_type (ActionType, optional): The type of the event being recorded.
+ action_type (str, optional): The type of the event being recorded.
Events default to 'action'. Other options include 'api' and 'llm'.
model (Models, optional): The model used during the event if an LLM is used (i.e. GPT-4).
For models, see the types available in the Models enum.
@@ -168,15 +175,15 @@ class Client:
# calculating price
action = action_type
if bool(model):
- action = ActionType.LLM
+ action = 'llm'
if not bool(prompt):
raise ValueError(
"Prompt is required when model is provided.")
# Throw error if action type is 'llm' but no model is specified
- if action == ActionType.LLM and not bool(model):
+ if action == 'llm' and not bool(model):
raise ValueError(
- f"`model` is a required parameter if `action_type` is set as {ActionType.LLM}. " +
+ f"`model` is a required parameter if `action_type` is set as `llm`. " +
f"Model can be set as: {list([mod.value for mod in Models])}")
try:
diff --git a/agentops/event.py b/agentops/event.py
index e069df91..1bc0b2ab 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -4,7 +4,7 @@ AgentOps events.
Classes:
Event: Represents discrete events to be recorded.
"""
-from .helpers import get_ISO_time, ActionType, Models
+from .helpers import get_ISO_time, Models
from typing import Optional, List
from pydantic import Field
@@ -18,7 +18,7 @@ class Event:
params (str, optional): The parameters passed to the operation.
returns (str, optional): The output of the operation.
result (str, optional): Result of the operation, e.g., "Success", "Fail", "Indeterminate". Defaults to "Indeterminate".
- action_type (ActionType, optional): Type of action of the event e.g. 'action', 'llm', 'api'. Defaults to ActionType.ACTION.
+ action_type (str, optional): Type of action of the event e.g. 'action', 'llm', 'api'. Defaults to 'action'.
model (Models, optional): The model used during the event if an LLM is used (i.e. GPT-4).
For models, see the types available in the Models enum.
If a model is set but an action_type is not, the action_type will be coerced to 'llm'.
@@ -33,7 +33,7 @@ class Event:
params (str, optional): The parameters passed to the operation.
returns (str, optional): The output of the operation.
result (str): Result of the operation.
- action_type (ActionType): Type of action of the event.
+ action_type (str): Type of action of the event.
model (Models, optional): The model used during the event.
prompt (str, optional): The input prompt for an LLM call.
tags (List[str], optional): Tags associated with the event.
@@ -47,7 +47,9 @@ class Event:
result: str = Field("Indeterminate",
description="Result of the operation",
pattern="^(Success|Fail|Indeterminate)$"),
- action_type: Optional[ActionType] = ActionType.ACTION,
+ action_type: Optional[str] = Field("action",
+ description="Type of action that the user is recording",
+ pattern="^(action|api|llm)$"),
model: Optional[Models] = None,
prompt: Optional[str] = None,
tags: Optional[List[str]] = None,
diff --git a/agentops/helpers.py b/agentops/helpers.py
index e77d983b..25859cbc 100644
--- a/agentops/helpers.py
+++ b/agentops/helpers.py
@@ -19,9 +19,3 @@ class Models(Enum):
GPT_4_32K_0314 = "gpt-4-32k-0314"
GPT_4_0613 = "gpt-4-0613"
TEXT_EMBEDDING_ADA_002 = "text-embedding-ada-002"
-
-
-class ActionType:
- LLM = "llm"
- API = "api"
- ACTION = "action"
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
new file mode 100644
index 00000000..a4b3c535
--- /dev/null
+++ b/agentops/llm_tracker.py
@@ -0,0 +1,151 @@
+import asyncio
+import functools
+import inspect
+from importlib import import_module
+from .event import Event
+from .helpers import get_ISO_time
+
+
+class LlmTracker:
+ SUPPORTED_APIS = {
+ 'openai': (
+ "Edit.create",
+ "Completion.create",
+ "ChatCompletion.create",
+ "Edit.acreate",
+ "Completion.acreate",
+ "ChatCompletion.acreate",
+ )
+ }
+
+ def __init__(self, client):
+ self.client = client
+ self.event_stream = None
+
+ def parse_and_record_event(self, api, result, kwargs, init_timestamp):
+ if api == 'openai':
+ event = Event(
+ event_type=result.get('object'),
+ params=kwargs,
+ result='Success',
+ returns={"content":
+ result['choices'][0]['message']['content']},
+ action_type='llm',
+ model=result['model'],
+ prompt=kwargs['messages'],
+ init_timestamp=init_timestamp
+ )
+ self.client.record(event)
+
+ async def parse_and_record_async(self, api, result, kwargs, init_timestamp):
+ if api == 'openai':
+ event = Event(
+ event_type=result.get('object'),
+ params=kwargs,
+ result='Success',
+ returns={"content":
+ result['choices'][0]['message']['content']},
+ action_type='llm',
+ model=result['model'],
+ prompt=kwargs['messages'],
+ init_timestamp=init_timestamp
+ )
+ self.client.record(event)
+
+ def parse_and_record_chunks(self, api, result, kwargs, init_timestamp):
+ if api == 'openai':
+ model = result.get('model')
+ choices = result['choices']
+ token = choices[0]['delta'].get('content', '')
+ finish_reason = choices[0]['finish_reason']
+
+ if self.event_stream == None:
+ self.event_stream = Event(
+ event_type='openai stream',
+ params=kwargs,
+ result='Success',
+ returns={"finish_reason": None, "content": token},
+ action_type='llm',
+ model=model,
+ prompt='test prompt',
+ init_timestamp=init_timestamp
+ )
+ else:
+ self.event_stream.returns['content'] += token
+
+ # Finish reason is 'stop' or something else
+ if bool(finish_reason):
+ self.event_stream.returns['finish_reason'] = finish_reason
+ self.client.record(self.event_stream)
+ self.event_stream = None
+
+ def _override_method(self, api, original_method):
+ """
+ Generate a new method (either async or sync) that overrides the original
+ and records an event when called.
+ """
+
+ if inspect.iscoroutinefunction(original_method):
+ # Handle async generator for streams
+ @functools.wraps(original_method)
+ async def async_method(*args, **kwargs):
+ init_timestamp = get_ISO_time()
+ async_result = await original_method(*args, **kwargs)
+ # Async non-stream
+ try:
+ await self.parse_and_record_async(api, async_result,
+ kwargs, init_timestamp)
+ return async_result
+ # Async stream
+ except:
+ async def generator():
+ async for result in async_result:
+ self.parse_and_record_chunks(
+ api, result, kwargs, init_timestamp)
+ yield result
+
+ return generator()
+ return async_method
+
+ # Handle sync code
+ else:
+ @functools.wraps(original_method)
+ def sync_method(*args, **kwargs):
+ init_timestamp = get_ISO_time()
+ result = original_method(*args, **kwargs)
+ # Sync stream
+ try:
+ self.parse_and_record_event(
+ api, result, kwargs, init_timestamp)
+ return result
+ # Sync non-stream
+ except:
+ def generator():
+ for res in result:
+ self.parse_and_record_chunks(
+ api, res, kwargs, init_timestamp)
+ yield res
+
+ return generator()
+
+ return sync_method
+
+ def override_api(self, api):
+ """
+ Overrides key methods of the specified API to record events.
+ """
+ if api not in self.SUPPORTED_APIS:
+ raise ValueError(f"Unsupported API: {api}")
+
+ module = import_module(api)
+
+ for method_path in self.SUPPORTED_APIS[api]:
+ method_parts = method_path.split(".")
+ original_method = functools.reduce(getattr, method_parts, module)
+ new_method = self._override_method(api, original_method)
+
+ if len(method_parts) == 1:
+ setattr(module, method_parts[0], new_method)
+ else:
+ parent = functools.reduce(getattr, method_parts[:-1], module)
+ setattr(parent, method_parts[-1], new_method)
diff --git a/setup.py b/setup.py
index 0edceeb8..d26181ee 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@ setup(
author_email='[email protected]',
packages=find_packages(),
install_requires=[
- 'requests'
+ 'requests', 'pydantic'
],
extras_require={
'dev': [
diff --git a/tests/api_call_agent.py b/tests/api_call_agent.py
new file mode 100644
index 00000000..6f5397b7
--- /dev/null
+++ b/tests/api_call_agent.py
@@ -0,0 +1,18 @@
+import openai
+from openai import ChatCompletion
+
+import agentops
+
+ao_client = agentops.Client(api_key='',
+ endpoint="http://localhost:8000",
+ tags=['mock tests'])
+
+openai.api_key = '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'
+
+message = [{"role": "user", "content": "Hello"},
+ {"role": "assistant", "content": "Hi there!"}]
+
+ChatCompletion.create(
+ model='gpt-3.5-turbo', messages=message, temperature=0.5)
+
+ao_client.end_session('Success')
diff --git a/tests/async_call_agent.py b/tests/async_call_agent.py
new file mode 100644
index 00000000..4195e529
--- /dev/null
+++ b/tests/async_call_agent.py
@@ -0,0 +1,66 @@
+import agentops
+import asyncio
+import openai
+
+
+# from wandb.integration.openai import autolog
+# autolog()
+
+
+ao_client = agentops.Client(api_key='',
+ tags=['mock tests'])
+
+
+openai.api_key = '3'
+
+
+async def stream_achat():
+ message = [{"role": "user", "content": "stream achat test"}]
+
+ res = await openai.ChatCompletion.acreate(
+ model='gpt-3.5-turbo', messages=message, temperature=0.5, stream=True)
+ async for r in res:
+ print(r)
+ print('*'*100)
+
+
+def stream_chat():
+ message = [{"role": "user", "content": "stream chat test"}]
+ res = openai.ChatCompletion.create(
+ model='gpt-3.5-turbo', messages=message, temperature=0.5, stream=True)
+ for r in res:
+ print(r)
+ print('*'*100)
+
+
+async def achat():
+ message = [{"role": "user", "content": "achat test"}]
+ res = await openai.ChatCompletion.acreate(
+ model='gpt-3.5-turbo', messages=message, temperature=0.5)
+ print(res)
+
+ print('*'*100)
+
+
+def chat():
+ message = [{"role": "user", "content": "chat test"}]
+ res = openai.ChatCompletion.create(
+ model='gpt-3.5-turbo', messages=message, temperature=0.5)
+ print(res)
+
+ print('*'*100)
+
+
+print('running async stream')
+asyncio.run(stream_achat())
+
+print('running sync stream')
+stream_chat()
+
+print('running async chat')
+asyncio.run(achat())
+
+print('running sync chat')
+chat()
+
+ao_client.end_session('Success')
diff --git a/tox.ini b/tox.ini
index 37402e4f..25fac5bf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,7 +11,9 @@ deps =
pytest
requests_mock
coverage
+ pydantic
mypy: mypy
+ types-requests
commands =
coverage run --source agentops -m pytest
coverage report -m