Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Working state llm_tracker
Howard Gil7e67cfb79444
diff --git a/agentops/client.py b/agentops/client.pyindex b56d2777..1342c454 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -5,7 +5,7 @@ Classes:Client: Provides methods to interact with the AgentOps service."""-from .event import Event+from .event import Event, ActionEvent, ErrorEventfrom .helpers import get_ISO_time, singleton, check_call_stack_for_agent_idfrom .session import Sessionfrom .worker import Worker@@ -104,8 +104,6 @@ class Client(metaclass=MetaClient):event (Event): The event to record."""- agent_id = check_call_stack_for_agent_id()- event.agent_id = agent_idif self._session is not None and not self._session.has_ended:self._worker.add_event(event.__dict__)else:@@ -160,23 +158,19 @@ class Client(metaclass=MetaClient):returns = list(returns)# Record the event after the function call- self.record(Event(event_type=event_name,- params=arg_values,- returns=returns,- result='Success',- action_type='action',- init_timestamp=init_time,- tags=tags))+ self.record(ActionEvent(tags=tags,+ params=arg_values,+ init_timestamp=init_time,+ action_type=event_name,+ agent_id=check_call_stack_for_agent_id(),+ details=returns))except Exception as e:# Record the event after the function call- self.record(Event(event_type=event_name,- params=arg_values,- returns={f"{type(e).__name__}": str(e)},- result='Fail',- action_type='action',- init_timestamp=init_time,- tags=tags))+ self.record(ErrorEvent(tags=tags,+ params=arg_values,+ init_timestamp=init_time,+ details={f"{type(e).__name__}": str(e)}))# Re-raise the exceptionraise@@ -194,7 +188,6 @@ class Client(metaclass=MetaClient):# Update with positional argumentsarg_values.update(dict(zip(arg_names, args)))arg_values.update(kwargs)- agent_id = check_call_stack_for_agent_id()try:@@ -205,25 +198,19 @@ class Client(metaclass=MetaClient):returns = list(returns)# Record the event after the function call- self.record(Event(event_type=event_name,- params=arg_values,- returns=returns,- result='Success',- action_type='action',- init_timestamp=init_time,- agent_id=agent_id,- tags=tags))+ self.record(ActionEvent(tags=tags,+ params=arg_values,+ init_timestamp=init_time,+ agent_id=check_call_stack_for_agent_id(),+ action_type=event_name,+ details=returns))except Exception as e:# Record the event after the function call- self.record(Event(event_type=event_name,- params=arg_values,- returns={f"{type(e).__name__}": str(e)},- result='Fail',- action_type='action',- agent_id=agent_id,- init_timestamp=init_time,- tags=tags))+ self.record(ErrorEvent(tags=tags,+ params=arg_values,+ init_timestamp=init_time,+ details={f"{type(e).__name__}": str(e)}))# Re-raise the exceptionraisediff --git a/agentops/event.py b/agentops/event.pyindex 57456f63..b8fd957d 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -27,7 +27,7 @@ class ActionEvent(Event):# TODO: Should not be optional, but non-default argument 'agent_id' follows default argument erroragent_id: Optional[UUID] = Noneaction_type: Optional[str] = None- detail: Optional[str] = None+ details: Optional[str] = Nonelogs: Optional[str] = Nonescreenshot: Optional[str] = Nonediff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 9a63f417..2be31223 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -5,6 +5,7 @@ from packaging.version import parseimport loggingfrom .event import LLMEventfrom .helpers import get_ISO_time, check_call_stack_for_agent_id+import inspectclass LlmTracker:@@ -38,7 +39,7 @@ class LlmTracker:if self.event_stream == None:self.event_stream = LLMEvent(init_timestamp=init_timestamp,- params=kwargs["invocation_params"],+ params=kwargs,agent_id=check_call_stack_for_agent_id(),prompt=kwargs["messages"],completion={"finish_reason": None,@@ -57,8 +58,9 @@ class LlmTracker:self.client.record(self.event_stream)self.event_stream = Noneexcept Exception as e:+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif inspect.isasyncgen(response):@@ -81,7 +83,7 @@ class LlmTracker:try:self.client.record(LLMEvent(init_timestamp=init_timestamp,- params=kwargs["invocation_params"],+ params=kwargs,agent_id=check_call_stack_for_agent_id(),prompt=kwargs['messages'],completion={"content":@@ -92,12 +94,14 @@ class LlmTracker:completion_tokens=response.get('usage',{}).get('completion_tokens')))- except:++ except: # NOTE: Execution should never reach here. Should remove if does not break anything+# v1.0.0+ responses are objectstry:self.client.record(LLMEvent(init_timestamp=init_timestamp,- params=kwargs["invocation_params"],+ params=kwargs,agent_id=check_call_stack_for_agent_id(),# TODO: Will need to make the completion the key for content, splat out the model dumpprompt=kwargs['messages'],@@ -107,10 +111,11 @@ class LlmTracker:prompt_tokens=response.usage.prompt_tokens,completion_tokens=response.usage.completion_tokens))- # Standard response+ # Standard responseexcept Exception as e:+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")return response@@ -157,8 +162,9 @@ class LlmTracker:self.client.record(self.event_stream)self.event_stream = Noneexcept Exception as e:+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different pathslogging.warning(- f"AgentOps Error: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif isinstance(response, Stream):@@ -200,8 +206,9 @@ class LlmTracker:))# Standard responseexcept Exception as e:- logging.error(- f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")+ # TODO: This error is specific to only one path of failure. Should be more generic or have different logging for different paths+ logging.warning(+ f"AgentOps: Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")return responsediff --git a/agentops/meta_client.py b/agentops/meta_client.pyindex 50bee809..9ae9f051 100644--- a/agentops/meta_client.py+++ b/agentops/meta_client.py@@ -5,6 +5,7 @@ import tracebackfrom .host_env import get_host_envfrom .http_client import HttpClientfrom .helpers import safe_serialize+from importlib.metadata import versionclass MetaClient(type):@@ -25,7 +26,7 @@ class MetaClient(type):exception_message = str(exception)exception_traceback = traceback.format_exc()developer_error = {- "sdk_version": read_version_from_pyproject(),+ "sdk_version": get_agentops_version(),"type": exception_type,"message": exception_message,"stack_trace": exception_traceback,@@ -50,7 +51,10 @@ def handle_exceptions(method):return wrapper-def read_version_from_pyproject():- with open("../pyproject.toml", "r") as pyproject_file:- pyproject_contents = toml.load(pyproject_file)- return pyproject_contents['project']['version']+def get_agentops_version():+ try:+ pkg_version = version("agentops")+ return pkg_version+ except Exception as e:+ print(f"Error reading package version: {e}")+ return Nonediff --git a/tests/test_llm_tracker_ge_1_async.py b/tests/test_llm_tracker_ge_1_async.pynew file mode 100644index 00000000..3617cc3b--- /dev/null+++ b/tests/test_llm_tracker_ge_1_async.py@@ -0,0 +1,38 @@+from openai import AsyncOpenAI # Assuming AsyncOpenAI is the async version of the client+import asyncio+import agentops+from packaging.version import parse+from importlib import import_module+import sys++ao_client = agentops.Client()+++async def call_openai_async():+ client = AsyncOpenAI() # Using the async client++ response = await client.chat.completions.create( # Await the async call+ model="gpt-3.5-turbo",+ messages=[+ {+ "role": "system",+ "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."+ },+ {+ "role": "user",+ "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"+ }+ ],+ temperature=0.7,+ max_tokens=64,+ top_p=1+ )++ print(response)+++async def main():+ await call_openai_async()+ ao_client.end_session('Success') # This would also need to be made async if it makes network calls++asyncio.run(main())diff --git a/tests/test_llm_tracker_ge_1_sync.py b/tests/test_llm_tracker_ge_1_sync.pynew file mode 100644index 00000000..9b69c19b--- /dev/null+++ b/tests/test_llm_tracker_ge_1_sync.py@@ -0,0 +1,45 @@+from openai import OpenAI+import agentops+from packaging.version import parse+from importlib import import_module+import sys+++ao_client = agentops.Client()++api = 'openai'+if api in sys.modules:+ module = import_module(api)+ if api == 'openai':+ if hasattr(module, '__version__'):+ module_version = parse(module.__version__)+ print('openai version: ', module_version)+++@ao_client.record_function('sample function being record')+def call_openai():+ client = OpenAI()++ response = client.chat.completions.create(+ model="gpt-3.5-turbo",+ messages=[+ {+ "role": "system",+ "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."+ },+ {+ "role": "user",+ "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"+ }+ ],+ temperature=0.7,+ max_tokens=64,+ top_p=1+ )++ print(response)+++call_openai()++ao_client.end_session('Success')diff --git a/tests/test_llm_tracker_lt_1_async.py b/tests/test_llm_tracker_lt_1_async.pynew file mode 100644index 00000000..46d8c513--- /dev/null+++ b/tests/test_llm_tracker_lt_1_async.py@@ -0,0 +1 @@+# ?diff --git a/tests/test_llm_tracker_lt_1_sync.py b/tests/test_llm_tracker_lt_1_sync.pynew file mode 100644index 00000000..81ccac73--- /dev/null+++ b/tests/test_llm_tracker_lt_1_sync.py@@ -0,0 +1,34 @@+import openai+from openai import ChatCompletion+import agentops+from packaging.version import parse+from importlib import import_module+import sys+++ao_client = agentops.Client()++api = 'openai'+if api in sys.modules:+ module = import_module(api)+ if api == 'openai':+ if hasattr(module, '__version__'):+ module_version = parse(module.__version__)+ print('openai version: ', module_version)+++@ao_client.record_function('sample function being record')+def call_openai():++ message = [{"role": "user", "content": "Hello"},+ {"role": "assistant", "content": "Hi there!"}]++ response = ChatCompletion.create(+ model='gpt-3.5-turbo', messages=message, temperature=0.5)++ print(response)+++call_openai()++ao_client.end_session('Success')diff --git a/tests/test_teardown.py b/tests/test_teardown.pyindex e3f0d902..06bcceb4 100644--- a/tests/test_teardown.py+++ b/tests/test_teardown.py@@ -2,7 +2,7 @@ import pytestimport requests_mockimport sys-from agentops import Client, Event+from agentops import Client@pytest.fixture