Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Autogen support (#204)
Braelyn Boyntonea6328df1f03
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex efbfd365..02324a41 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -1,7 +1,5 @@# agentops/__init__.py-from os import environfrom typing import Optional, List, Union-from .client import Clientfrom .config import Configurationfrom .event import Event, ActionEvent, LLMEvent, ToolEvent, ErrorEvent@@ -14,6 +12,7 @@ try:except ModuleNotFoundError:pass+def init(api_key: Optional[str] = None,parent_key: Optional[str] = None,endpoint: Optional[str] = None,@@ -134,3 +133,7 @@ def set_parent_key(parent_key):def stop_instrumenting():Client().stop_instrumenting()+++def create_agent(name: str, agent_id: Optional[str] = None):+ return Client().create_agent(name=name, agent_id=agent_id)diff --git a/agentops/client.py b/agentops/client.pyindex 5447f510..92653f1e 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -5,9 +5,11 @@Client: Provides methods to interact with the AgentOps service."""import os+import uuid+from .event import ActionEvent, ErrorEvent, Eventfrom .enums import EndState-from .helpers import get_ISO_time, singleton, check_call_stack_for_agent_id+from .helpers import get_ISO_time, singleton, check_call_stack_for_agent_id, get_partner_frameworksfrom .session import Sessionfrom .worker import Workerfrom .host_env import get_host_env@@ -37,8 +39,8 @@ class Client(metaclass=MetaClient):api_key (str, optional): API Key for AgentOps services. If none is provided, key willbe read from the AGENTOPS_API_KEY environment variable.- parent_key (str, optional): Organization key to give visibility of all user sessions the user's organization. If none is provided, key will- be read from the AGENTOPS_PARENT_KEY environment variable.+ parent_key (str, optional): Organization key to give visibility of all user sessions the user's organization.+ If none is provided, key will be read from the AGENTOPS_PARENT_KEY environment variable.endpoint (str, optional): The endpoint for the AgentOps service. If none is provided, key willbe read from the AGENTOPS_API_ENDPOINT environment variable. Defaults to 'https://api.agentops.ai'.max_wait_time (int, optional): The maximum time to wait in milliseconds before flushing the queue.@@ -46,13 +48,15 @@ class Client(metaclass=MetaClient):max_queue_size (int, optional): The maximum size of the event queue. Defaults to 100.tags (List[str], optional): Tags for the sessions that can be used for grouping orsorting later (e.g. ["GPT-4"]).- override (bool, optional): [Deprecated] Use `instrument_llm_calls` instead. Whether to instrument LLM calls and emit LLMEvents..+ override (bool, optional): [Deprecated] Use `instrument_llm_calls` instead. Whether to instrument LLM calls+ and emit LLMEvents.instrument_llm_calls (bool): Whether to instrument LLM calls and emit LLMEvents..auto_start_session (bool): Whether to start a session automatically when the client is created.inherited_session_id (optional, str): Init Agentops with an existing SessionAttributes:_session (Session, optional): A Session is a grouping of events (e.g. a run of your agent).- _worker (Worker, optional): A Worker manages the event queue and sends session updates to the AgentOps api server+ _worker (Worker, optional): A Worker manages the event queue and sends session updates to the AgentOps api+ server"""def __init__(self,@@ -78,6 +82,7 @@ class Client(metaclass=MetaClient):self._tags: Optional[List[str]] = tagsself._tags_for_future_session: Optional[List[str]] = None+self._env_data_opt_out = os.getenv('AGENTOPS_ENV_DATA_OPT_OUT') and os.getenv('AGENTOPS_ENV_DATA_OPT_OUT').lower() == 'true'@@ -92,6 +97,8 @@ class Client(metaclass=MetaClient):self._handle_unclean_exits()+ instrument_llm_calls, auto_start_session = self._check_for_partner_frameworks(instrument_llm_calls, auto_start_session)+if auto_start_session:self.start_session(tags, self.config, inherited_session_id)else:@@ -101,6 +108,19 @@ class Client(metaclass=MetaClient):self.llm_tracker = LlmTracker(self)self.llm_tracker.override_api()+ def _check_for_partner_frameworks(self, instrument_llm_calls, auto_start_session) -> (bool, bool):+ partner_frameworks = get_partner_frameworks()+ for framework in partner_frameworks.keys():+ if framework in sys.modules:+ self.add_tags([framework])+ if 'autogen':+ import autogen+ autogen.runtime_logging.start(logger_type="agentops")++ return partner_frameworks[framework]++ return instrument_llm_calls, auto_start_session+def add_tags(self, tags: List[str]):"""Append to session tags at runtime.@@ -108,15 +128,25 @@ class Client(metaclass=MetaClient):Args:tags (List[str]): The list of tags to append."""- if self._session.tags is not None:- for tag in tags:- if tag not in self._session.tags:- self._session.tags.append(tag)- else:- self._session.tags = tags- if self._session is not None and self._worker is not None:- self._worker.update_session(self._session)+ if self._session:+ if self._session.tags is not None:+ for tag in tags:+ if tag not in self._session.tags:+ self._session.tags.append(tag)+ else:+ self._session.tags = tags++ if self._session is not None and self._worker is not None:+ self._worker.update_session(self._session)++ else:+ if self._tags_for_future_session:+ for tag in tags:+ if tag not in self._session.tags:+ self._tags_for_future_session.append(tag)+ else:+ self._tags_for_future_session = tagsdef set_tags(self, tags: List[str]):"""@@ -253,7 +283,7 @@ class Client(metaclass=MetaClient):start_session_result = self._worker.start_session(self._session)if not start_session_result:self._session = None- return logger.warning("🖇 AgentOps: Cannot start session")+ return logger.warning("🖇 AgentOps: Cannot start session - No server response")logger.info('View info on this session at https://app.agentops.ai/drilldown?session_id=%s',self._session.session_id)@@ -294,9 +324,12 @@ class Client(metaclass=MetaClient):self._session = Noneself._worker = None- def create_agent(self, agent_id: str, name: str):+ def create_agent(self, name: str, agent_id: Optional[str] = None) -> str:+ if agent_id is None:+ agent_id = str(uuid.uuid4())if self._worker:self._worker.create_agent(agent_id, name)+ return agent_iddef _handle_unclean_exits(self):def cleanup(end_state: str = 'Fail', end_state_reason: Optional[str] = None):diff --git a/agentops/helpers.py b/agentops/helpers.pyindex 1a8cfcad..6565a24f 100644--- a/agentops/helpers.py+++ b/agentops/helpers.py@@ -12,6 +12,13 @@ import osfrom importlib.metadata import version+PARTNER_FRAMEWORKS = {+ # framework : instrument_llm_calls, auto_start_session+ 'autogen': (False, True),+ 'crewai': (False, True)+}++def singleton(class_):instances = {}@@ -138,3 +145,7 @@ def debug_print_function_params(func):return func(self, *args, **kwargs)return wrapper+++def get_partner_frameworks():+ return PARTNER_FRAMEWORKSdiff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex 250aa41c..9beb42ef 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -521,4 +521,4 @@ class LlmTracker:def undo_override_openai_v1_async_completion(self):global original_create_asyncfrom openai.resources.chat import completions- original_create_async = completions.AsyncCompletions.create+ completions.AsyncCompletions.create = original_create_asyncdiff --git a/pyproject.toml b/pyproject.tomlindex a81826f8..68b447c3 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -30,7 +30,7 @@ dev = ["requests_mock==1.11.0"]langchain = [- "langchain>=0.0.354"+ "langchain~=1.19"][project.urls]diff --git a/tests/core_manual_tests/upsert_events.py.ipynb b/tests/core_manual_tests/upsert_events.py.ipynbnew file mode 100644index 00000000..434008ae--- /dev/null+++ b/tests/core_manual_tests/upsert_events.py.ipynb@@ -0,0 +1,50 @@+{+ "cells": [+ {+ "cell_type": "code",+ "execution_count": 2,+ "id": "initial_id",+ "metadata": {+ "collapsed": true,+ "ExecuteTime": {+ "end_time": "2024-05-07T01:49:47.567382Z",+ "start_time": "2024-05-07T01:49:47.148365Z"+ }+ },+ "outputs": [],+ "source": [+ "import agentops"+ ]+ },+ {+ "cell_type": "code",+ "outputs": [],+ "source": [],+ "metadata": {+ "collapsed": false+ },+ "id": "cc69b52023168f58"+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": "Python 3",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 2+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython2",+ "version": "2.7.6"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 5+}