Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Working state llm_tracker

Howard Gil7e67cfb79444
diff --git a/agentops/client.py b/agentops/client.py
index b56d2777..1342c454 100644
--- a/agentops/client.py
+++ b/agentops/client.py
@@ -5,7 +5,7 @@ Classes:
Client: Provides methods to interact with the AgentOps service.
"""
-from .event import Event
+from .event import Event, ActionEvent, ErrorEvent
from .helpers import get_ISO_time, singleton, check_call_stack_for_agent_id
from .session import Session
from .worker import Worker
@@ -104,8 +104,6 @@ class Client(metaclass=MetaClient):
event (Event): The event to record.
"""
- agent_id = check_call_stack_for_agent_id()
- event.agent_id = agent_id
if self._session is not None and not self._session.has_ended:
self._worker.add_event(event.__dict__)
else:
@@ -160,23 +158,19 @@ class Client(metaclass=MetaClient):
returns = list(returns)
# Record the event after the function call
- self.record(Event(event_type=event_name,
- params=arg_values,
- returns=returns,
- result='Success',
- action_type='action',
- init_timestamp=init_time,
- tags=tags))
+ self.record(ActionEvent(tags=tags,
+ params=arg_values,
+ init_timestamp=init_time,
+ action_type=event_name,
+ agent_id=check_call_stack_for_agent_id(),
+ details=returns))
except Exception as e:
# Record the event after the function call
- self.record(Event(event_type=event_name,
- params=arg_values,
- returns={f"{type(e).__name__}": str(e)},
- result='Fail',
- action_type='action',
- init_timestamp=init_time,
- tags=tags))
+ self.record(ErrorEvent(tags=tags,
+ params=arg_values,
+ init_timestamp=init_time,
+ details={f"{type(e).__name__}": str(e)}))
# Re-raise the exception
raise
@@ -194,7 +188,6 @@ class Client(metaclass=MetaClient):
# Update with positional arguments
arg_values.update(dict(zip(arg_names, args)))
arg_values.update(kwargs)
- agent_id = check_call_stack_for_agent_id()
try:
@@ -205,25 +198,19 @@ class Client(metaclass=MetaClient):
returns = list(returns)
# Record the event after the function call
- self.record(Event(event_type=event_name,
- params=arg_values,
- returns=returns,
- result='Success',
- action_type='action',
- init_timestamp=init_time,
- agent_id=agent_id,
- tags=tags))
+ self.record(ActionEvent(tags=tags,
+ params=arg_values,
+ init_timestamp=init_time,
+ agent_id=check_call_stack_for_agent_id(),
+ action_type=event_name,
+ details=returns))
except Exception as e:
# Record the event after the function call
- self.record(Event(event_type=event_name,
- params=arg_values,
- returns={f"{type(e).__name__}": str(e)},
- result='Fail',
- action_type='action',
- agent_id=agent_id,
- init_timestamp=init_time,
- tags=tags))
+ self.record(ErrorEvent(tags=tags,
+ params=arg_values,
+ init_timestamp=init_time,
+ details={f"{type(e).__name__}": str(e)}))
# Re-raise the exception
raise
diff --git a/agentops/event.py b/agentops/event.py
index 57456f63..b8fd957d 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -27,7 +27,7 @@ class ActionEvent(Event):
# TODO: Should not be optional, but non-default argument 'agent_id' follows default argument error
agent_id: Optional[UUID] = None
action_type: Optional[str] = None
- detail: Optional[str] = None
+ details: Optional[str] = None
logs: Optional[str] = None
screenshot: Optional[str] = None
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
index 9a63f417..2be31223 100644
--- a/agentops/llm_tracker.py
+++ b/agentops/llm_tracker.py
@@ -5,6 +5,7 @@ from packaging.version import parse
import logging
from .event import LLMEvent
from .helpers import get_ISO_time, check_call_stack_for_agent_id
+import inspect
class LlmTracker:
@@ -38,7 +39,7 @@ class LlmTracker:
if self.event_stream == None:
self.event_stream = LLMEvent(
init_timestamp=init_timestamp,
- params=kwargs["invocation_params"],
+ params=kwargs,
agent_id=check_call_stack_for_agent_id(),
prompt=kwargs["messages"],
completion={"finish_reason": None,
@@ -57,8 +58,9 @@ class LlmTracker:
self.client.record(self.event_stream)
self.event_stream = None
except Exception as e:
+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
if inspect.isasyncgen(response):
@@ -81,7 +83,7 @@ class LlmTracker:
try:
self.client.record(LLMEvent(
init_timestamp=init_timestamp,
- params=kwargs["invocation_params"],
+ params=kwargs,
agent_id=check_call_stack_for_agent_id(),
prompt=kwargs['messages'],
completion={"content":
@@ -92,12 +94,14 @@ class LlmTracker:
completion_tokens=response.get('usage',
{}).get('completion_tokens')
))
- except:
+
+ except: # NOTE: Execution should never reach here. Should remove if does not break anything
+
# v1.0.0+ responses are objects
try:
self.client.record(LLMEvent(
init_timestamp=init_timestamp,
- params=kwargs["invocation_params"],
+ params=kwargs,
agent_id=check_call_stack_for_agent_id(),
# TODO: Will need to make the completion the key for content, splat out the model dump
prompt=kwargs['messages'],
@@ -107,10 +111,11 @@ class LlmTracker:
prompt_tokens=response.usage.prompt_tokens,
completion_tokens=response.usage.completion_tokens
))
- # Standard response
+ # Standard response
except Exception as e:
+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
return response
@@ -157,8 +162,9 @@ class LlmTracker:
self.client.record(self.event_stream)
self.event_stream = None
except Exception as e:
+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
logging.warning(
- f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
if isinstance(response, Stream):
@@ -200,8 +206,9 @@ class LlmTracker:
))
# Standard response
except Exception as e:
- logging.error(
- f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths
+ logging.warning(
+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
return response
diff --git a/agentops/meta_client.py b/agentops/meta_client.py
index 50bee809..9ae9f051 100644
--- a/agentops/meta_client.py
+++ b/agentops/meta_client.py
@@ -5,6 +5,7 @@ import traceback
from .host_env import get_host_env
from .http_client import HttpClient
from .helpers import safe_serialize
+from importlib.metadata import version
class MetaClient(type):
@@ -25,7 +26,7 @@ class MetaClient(type):
exception_message = str(exception)
exception_traceback = traceback.format_exc()
developer_error = {
- "sdk_version": read_version_from_pyproject(),
+ "sdk_version": get_agentops_version(),
"type": exception_type,
"message": exception_message,
"stack_trace": exception_traceback,
@@ -50,7 +51,10 @@ def handle_exceptions(method):
return wrapper
-def read_version_from_pyproject():
- with open("../pyproject.toml", "r") as pyproject_file:
- pyproject_contents = toml.load(pyproject_file)
- return pyproject_contents['project']['version']
+def get_agentops_version():
+ try:
+ pkg_version = version("agentops")
+ return pkg_version
+ except Exception as e:
+ print(f"Error reading package version: {e}")
+ return None
diff --git a/tests/test_llm_tracker_ge_1_async.py b/tests/test_llm_tracker_ge_1_async.py
new file mode 100644
index 00000000..3617cc3b
--- /dev/null
+++ b/tests/test_llm_tracker_ge_1_async.py
@@ -0,0 +1,38 @@
+from openai import AsyncOpenAI # Assuming AsyncOpenAI is the async version of the client
+import asyncio
+import agentops
+from packaging.version import parse
+from importlib import import_module
+import sys
+
+ao_client = agentops.Client()
+
+
+async def call_openai_async():
+ client = AsyncOpenAI() # Using the async client
+
+ response = await client.chat.completions.create( # Await the async call
+ model="gpt-3.5-turbo",
+ messages=[
+ {
+ "role": "system",
+ "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."
+ },
+ {
+ "role": "user",
+ "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"
+ }
+ ],
+ temperature=0.7,
+ max_tokens=64,
+ top_p=1
+ )
+
+ print(response)
+
+
+async def main():
+ await call_openai_async()
+ ao_client.end_session('Success') # This would also need to be made async if it makes network calls
+
+asyncio.run(main())
diff --git a/tests/test_llm_tracker_ge_1_sync.py b/tests/test_llm_tracker_ge_1_sync.py
new file mode 100644
index 00000000..9b69c19b
--- /dev/null
+++ b/tests/test_llm_tracker_ge_1_sync.py
@@ -0,0 +1,45 @@
+from openai import OpenAI
+import agentops
+from packaging.version import parse
+from importlib import import_module
+import sys
+
+
+ao_client = agentops.Client()
+
+api = 'openai'
+if api in sys.modules:
+ module = import_module(api)
+ if api == 'openai':
+ if hasattr(module, '__version__'):
+ module_version = parse(module.__version__)
+ print('openai version: ', module_version)
+
+
+@ao_client.record_function('sample function being record')
+def call_openai():
+ client = OpenAI()
+
+ response = client.chat.completions.create(
+ model="gpt-3.5-turbo",
+ messages=[
+ {
+ "role": "system",
+ "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."
+ },
+ {
+ "role": "user",
+ "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"
+ }
+ ],
+ temperature=0.7,
+ max_tokens=64,
+ top_p=1
+ )
+
+ print(response)
+
+
+call_openai()
+
+ao_client.end_session('Success')
diff --git a/tests/test_llm_tracker_lt_1_async.py b/tests/test_llm_tracker_lt_1_async.py
new file mode 100644
index 00000000..46d8c513
--- /dev/null
+++ b/tests/test_llm_tracker_lt_1_async.py
@@ -0,0 +1 @@
+# ?
diff --git a/tests/test_llm_tracker_lt_1_sync.py b/tests/test_llm_tracker_lt_1_sync.py
new file mode 100644
index 00000000..81ccac73
--- /dev/null
+++ b/tests/test_llm_tracker_lt_1_sync.py
@@ -0,0 +1,34 @@
+import openai
+from openai import ChatCompletion
+import agentops
+from packaging.version import parse
+from importlib import import_module
+import sys
+
+
+ao_client = agentops.Client()
+
+api = 'openai'
+if api in sys.modules:
+ module = import_module(api)
+ if api == 'openai':
+ if hasattr(module, '__version__'):
+ module_version = parse(module.__version__)
+ print('openai version: ', module_version)
+
+
+@ao_client.record_function('sample function being record')
+def call_openai():
+
+ message = [{"role": "user", "content": "Hello"},
+ {"role": "assistant", "content": "Hi there!"}]
+
+ response = ChatCompletion.create(
+ model='gpt-3.5-turbo', messages=message, temperature=0.5)
+
+ print(response)
+
+
+call_openai()
+
+ao_client.end_session('Success')
diff --git a/tests/test_teardown.py b/tests/test_teardown.py
index e3f0d902..06bcceb4 100644
--- a/tests/test_teardown.py
+++ b/tests/test_teardown.py
@@ -2,7 +2,7 @@ import pytest
import requests_mock
import sys
-from agentops import Client, Event
+from agentops import Client
@pytest.fixture