Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Fix autogen reload (#468)
Alex Reibmanf4d834780f42
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 4c8f8dae..ccb5427d 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -22,6 +22,7 @@ except ModuleNotFoundError:if "autogen" in sys.modules:Client().configure(instrument_llm_calls=False)+ Client()._initialize_autogen_logger()Client().add_default_tags(["autogen"])if "crewai" in sys.modules:@@ -72,7 +73,6 @@ def init(Client().unsuppress_logs()t = threading.Thread(target=check_agentops_update)t.start()-if Client().is_initialized:return logger.warning("AgentOps has already been initialized. If you are trying to start a session, call agentops.start_session() instead."@@ -101,7 +101,6 @@ def init("auto_start_session is set to False - inherited_session_id will not be used to automatically start a session")return Client().initialize()-Client().configure(auto_start_session=False)Client().initialize()return Client().start_session(inherited_session_id=inherited_session_id)diff --git a/agentops/client.py b/agentops/client.pyindex 0c41f0c9..80cb1cc5 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -87,7 +87,6 @@ class Client(metaclass=MetaClient):returnself.unsuppress_logs()-if self._config.api_key is None:return logger.error("Could not initialize AgentOps client - API Key is missing."@@ -95,8 +94,6 @@ class Client(metaclass=MetaClient):)self._handle_unclean_exits()- self._initialize_partner_framework()-self._initialized = Trueif self._config.instrument_llm_calls:@@ -116,7 +113,7 @@ class Client(metaclass=MetaClient):return session- def _initialize_partner_framework(self) -> None:+ def _initialize_autogen_logger(self) -> None:try:import autogenfrom .partners.autogen_logger import AutogenLogger@@ -305,7 +302,8 @@ class Client(metaclass=MetaClient):self._pre_init_queue["agents"].append({"name": name, "agent_id": agent_id})- session.create_agent(name=name, agent_id=agent_id)+ else:+ session.create_agent(name=name, agent_id=agent_id)return agent_iddiff --git a/agentops/helpers.py b/agentops/helpers.pyindex fef08a54..fbcd21d5 100644--- a/agentops/helpers.py+++ b/agentops/helpers.py@@ -3,7 +3,7 @@ from functools import wrapsfrom datetime import datetime, timezoneimport inspectfrom typing import Union-import http.client+import requestsimport jsonfrom importlib.metadata import version, PackageNotFoundError@@ -127,25 +127,25 @@ def get_agentops_version():def check_agentops_update():- # using http.client to avoid this call being caught by requests_mock on tests- conn = http.client.HTTPSConnection("pypi.org")- conn.request("GET", "/pypi/agentops/json")- response = conn.getresponse()- data = response.read().decode()- json_data = json.loads(data)+ try:+ response = requests.get("https://pypi.org/pypi/agentops/json")- if response.status == 200:- latest_version = json_data["info"]["version"]+ if response.status_code == 200:+ json_data = response.json()+ latest_version = json_data["info"]["version"]- try:- current_version = version("agentops")- except PackageNotFoundError:- return None-- if not latest_version == current_version:- logger.warning(- f" WARNING: agentops is out of date. Please update with the command: 'pip install --upgrade agentops'"- )+ try:+ current_version = version("agentops")+ except PackageNotFoundError:+ return None++ if not latest_version == current_version:+ logger.warning(+ f" WARNING: agentops is out of date. Please update with the command: 'pip install --upgrade agentops'"+ )+ except Exception as e:+ logger.debug(f"Failed to check for updates: {e}")+ return None# Function decorator that prints function name and its arguments to the console for debug purposesdiff --git a/agentops/host_env.py b/agentops/host_env.pyindex d446d347..e466c86e 100644--- a/agentops/host_env.py+++ b/agentops/host_env.py@@ -47,13 +47,11 @@ def get_sys_packages():def get_installed_packages():-try:return {- # TODO: test# TODO: add to opt out"Installed Packages": {- dist.metadata["Name"]: dist.version+ dist.metadata.get("Name"): dist.metadata.get("Version")for dist in importlib.metadata.distributions()}}diff --git a/pyproject.toml b/pyproject.tomlindex 7b8aab8d..65e2e555 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"[project]name = "agentops"-version = "0.3.14"+version = "0.3.15rc1"authors = [{ name="Alex Reibman", email="[email protected]" },{ name="Shawn Qiu", email="[email protected]" },@@ -45,3 +45,7 @@ max_line_length = 120[project.scripts]agentops = "agentops.cli:main"++[tool.pytest.ini_options]+asyncio_mode = "strict"+asyncio_default_fixture_loop_scope = "function"diff --git a/tests/test_agent.py b/tests/test_agent.pyindex 408161b9..9005dc75 100644--- a/tests/test_agent.py+++ b/tests/test_agent.py@@ -1,13 +1,9 @@from unittest import TestCase-from agentops import track_agent-import agentopsclass TrackAgentTests(TestCase):def test_track_agent_with_class(self):- agentops.init()-@track_agent(name="agent_name")class TestAgentClass:t = "a"@@ -19,8 +15,6 @@ class TrackAgentTests(TestCase):self.assertIsNotNone(getattr(obj, "agent_ops_agent_id"))def test_track_agent_with_class_name(self):- agentops.init()-@track_agent(name="agent_name")class TestAgentClass:t = "a"diff --git a/tests/test_canary.py b/tests/test_canary.pyindex 8ff49a01..afe2a361 100644--- a/tests/test_canary.py+++ b/tests/test_canary.py@@ -13,16 +13,17 @@ def setup_teardown():agentops.end_all_sessions() # teardown part[email protected](autouse=True, scope="function")def mock_req():with requests_mock.Mocker() as m:url = "https://api.agentops.ai"- m.post(url + "/v2/create_events", text="ok")+ m.post(url + "/v2/create_events", json={"status": "ok"})m.post(url + "/v2/create_session", json={"status": "success", "jwt": "some_jwt"})m.post(url + "/v2/update_session", json={"status": "success", "token_cost": 5})- m.post(url + "/v2/developer_errors", text="ok")+ m.post(url + "/v2/developer_errors", json={"status": "ok"})+ m.post("https://pypi.org/pypi/agentops/json", status_code=404)yield m@@ -30,7 +31,7 @@ class TestCanary:def setup_method(self):self.url = "https://api.agentops.ai"self.api_key = "11111111-1111-4111-8111-111111111111"- agentops.init(api_key=self.api_key, max_wait_time=5, auto_start_session=False)+ agentops.init(api_key=self.api_key, max_wait_time=500, auto_start_session=False)def test_agent_ops_record(self, mock_req):# Arrange@@ -39,10 +40,11 @@ class TestCanary:# Actagentops.record(ActionEvent(event_type))- time.sleep(0.1)+ time.sleep(2)++ # 3 requests: check_for_updates, create_session, create_events+ assert len(mock_req.request_history) == 3- # Assert- assert len(mock_req.request_history) == 2request_json = mock_req.last_request.json()assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyassert request_json["events"][0]["event_type"] == event_typediff --git a/tests/test_events.py b/tests/test_events.pyindex a2d70d75..4c56fb8b 100644--- a/tests/test_events.py+++ b/tests/test_events.py@@ -13,16 +13,21 @@ def setup_teardown():agentops.end_all_sessions() # teardown part[email protected](autouse=True, scope="function")def mock_req():with requests_mock.Mocker() as m:url = "https://api.agentops.ai"- m.post(url + "/v2/create_events", text="ok")+ m.post(url + "/v2/create_events", json={"status": "ok"})m.post(url + "/v2/create_session", json={"status": "success", "jwt": "some_jwt"})m.post(url + "/v2/update_session", json={"status": "success", "token_cost": 5})- m.post(url + "/v2/developer_errors", text="ok")+ m.post(url + "/v2/developer_errors", json={"status": "ok"})+ m.post("https://pypi.org/pypi/agentops/json", status_code=404)++ m.post(+ url + "/v2/reauthorize_jwt", json={"status": "success", "jwt": "some_jwt"}+ )yield mdiff --git a/tests/test_pre_init.py b/tests/test_pre_init.pyindex f87219ac..4e729a57 100644--- a/tests/test_pre_init.py+++ b/tests/test_pre_init.py@@ -18,15 +18,19 @@ def setup_teardown():@contextlib.contextmanager[email protected](autouse=True)[email protected](autouse=True, scope="function")def mock_req():with requests_mock.Mocker() as m:url = "https://api.agentops.ai"- m.post(url + "/v2/create_agent", text="ok")- m.post(url + "/v2/update_session", text="ok")+ m.post(url + "/v2/create_agent", json={"status": "success"})+ m.post(url + "/v2/update_session", json={"status": "success", "token_cost": 5})m.post(url + "/v2/create_session", json={"status": "success", "jwt": "some_jwt"})+ m.post("https://pypi.org/pypi/agentops/json", status_code=404)++ m.post(url + "/v2/create_events", json={"status": "ok"})+ m.post(url + "/v2/developer_errors", json={"status": "ok"})yield m@@ -48,10 +52,20 @@ class TestPreInit:assert len(mock_req.request_history) == 0agentops.init(api_key=self.api_key)+ time.sleep(1)# Assert# start session and create agent- assert len(mock_req.request_history) == 2- assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key-agentops.end_session(end_state="Success")++ # Wait for flush+ time.sleep(1.5)++ # 4 requests: check_for_updates, create_session, create_agent, update_session+ assert len(mock_req.request_history) == 4++ assert (+ mock_req.request_history[-2].headers["X-Agentops-Api-Key"] == self.api_key+ )++ mock_req.reset()diff --git a/tests/test_record_action.py b/tests/test_record_action.pyindex 058e13dd..122a83d3 100644--- a/tests/test_record_action.py+++ b/tests/test_record_action.py@@ -18,11 +18,11 @@ def setup_teardown():@contextlib.contextmanager[email protected](autouse=True)[email protected](autouse=True, scope="function")def mock_req():with requests_mock.Mocker() as m:url = "https://api.agentops.ai"- m.post(url + "/v2/create_events", text="ok")+ m.post(url + "/v2/create_events", json={"status": "ok"})# Use iter to create an iterator that can return the jwt valuesjwt_tokens = iter(jwts)@@ -33,8 +33,10 @@ def mock_req():return {"status": "success", "jwt": next(jwt_tokens)}m.post(url + "/v2/create_session", json=create_session_response)+ m.post(url + "/v2/create_events", json={"status": "ok"})m.post(url + "/v2/update_session", json={"status": "success", "token_cost": 5})- m.post(url + "/v2/developer_errors", text="ok")+ m.post(url + "/v2/developer_errors", json={"status": "ok"})+ m.post("https://pypi.org/pypi/agentops/json", status_code=404)yield m@@ -44,7 +46,7 @@ class TestRecordAction:self.url = "https://api.agentops.ai"self.api_key = "11111111-1111-4111-8111-111111111111"self.event_type = "test_event_type"- agentops.init(self.api_key, max_wait_time=5, auto_start_session=False)+ agentops.init(self.api_key, max_wait_time=50, auto_start_session=False)def test_record_action_decorator(self, mock_req):agentops.start_session()@@ -57,8 +59,8 @@ class TestRecordAction:add_two(3, 4)time.sleep(0.1)- # Assert- assert len(mock_req.request_history) == 2+ # 3 requests: check_for_updates, start_session, record_action+ assert len(mock_req.request_history) == 3assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json["events"][0]["action_type"] == self.event_type@@ -78,8 +80,8 @@ class TestRecordAction:add_two(3, 4)time.sleep(0.1)- # Assert- assert len(mock_req.request_history) == 2+ # 3 requests: check_for_updates, start_session, record_action+ assert len(mock_req.request_history) == 3assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json["events"][0]["action_type"] == "add_two"@@ -98,14 +100,19 @@ class TestRecordAction:# Actadd_three(1, 2)- time.sleep(0.1)- add_three(1, 2)- time.sleep(0.1)+ add_three(1, 2, 4)- # Assert+ time.sleep(1.5)++ # 3 requests: check_for_updates, start_session, record_actionassert len(mock_req.request_history) == 3assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()++ assert request_json["events"][1]["action_type"] == self.event_type+ assert request_json["events"][1]["params"] == {"x": 1, "y": 2, "z": 4}+ assert request_json["events"][1]["returns"] == 7+assert request_json["events"][0]["action_type"] == self.event_typeassert request_json["events"][0]["params"] == {"x": 1, "y": 2, "z": 3}assert request_json["events"][0]["returns"] == 6@@ -128,7 +135,7 @@ class TestRecordAction:# Assertassert result == 7# Assert- assert len(mock_req.request_history) == 2+ assert len(mock_req.request_history) == 3assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json["events"][0]["action_type"] == self.event_type@@ -156,11 +163,10 @@ class TestRecordAction:# Actadd_three(1, 2, session=session_1)time.sleep(0.1)- add_three(1, 2, session=session_2)+ add_three(1, 2, 3, session=session_2)time.sleep(0.1)- # Assert- assert len(mock_req.request_history) == 4+ assert len(mock_req.request_history) == 5request_json = mock_req.last_request.json()assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key@@ -207,7 +213,7 @@ class TestRecordAction:time.sleep(0.1)# Assert- assert len(mock_req.request_history) == 4+ assert len(mock_req.request_history) == 5request_json = mock_req.last_request.json()assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keydiff --git a/tests/test_record_tool.py b/tests/test_record_tool.pyindex 29a5d473..d6b4745f 100644--- a/tests/test_record_tool.py+++ b/tests/test_record_tool.py@@ -23,7 +23,7 @@ def setup_teardown():def mock_req():with requests_mock.Mocker() as m:url = "https://api.agentops.ai"- m.post(url + "/v2/create_events", text="ok")+ m.post(url + "/v2/create_events", json={"status": "ok"})# Use iter to create an iterator that can return the jwt valuesjwt_tokens = iter(jwts)@@ -35,7 +35,7 @@ def mock_req():m.post(url + "/v2/create_session", json=create_session_response)m.post(url + "/v2/update_session", json={"status": "success", "token_cost": 5})- m.post(url + "/v2/developer_errors", text="ok")+ m.post(url + "/v2/developer_errors", json={"status": "ok"})yield m@@ -58,8 +58,8 @@ class TestRecordTool:add_two(3, 4)time.sleep(0.1)- # Assert- assert len(mock_req.request_history) == 2+ # 3 requests: check_for_updates, start_session, record_tool+ assert len(mock_req.request_history) == 3assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json["events"][0]["name"] == self.tool_name@@ -80,7 +80,7 @@ class TestRecordTool:time.sleep(0.1)# Assert- assert len(mock_req.request_history) == 2+ assert len(mock_req.request_history) == 3assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json["events"][0]["name"] == "add_two"@@ -103,8 +103,8 @@ class TestRecordTool:add_three(1, 2)time.sleep(0.1)- # Assert- assert len(mock_req.request_history) == 3+ # 4 requests: check_for_updates, start_session, record_tool, record_tool+ assert len(mock_req.request_history) == 4assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json["events"][0]["name"] == self.tool_name@@ -129,7 +129,7 @@ class TestRecordTool:# Assertassert result == 7# Assert- assert len(mock_req.request_history) == 2+ assert len(mock_req.request_history) == 3assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json["events"][0]["name"] == self.tool_name@@ -160,8 +160,8 @@ class TestRecordTool:add_three(1, 2, session=session_2)time.sleep(0.1)- # Assert- assert len(mock_req.request_history) == 4+ # 6 requests: check_for_updates, start_session, record_tool, start_session, record_tool, end_session+ assert len(mock_req.request_history) == 5request_json = mock_req.last_request.json()assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_key@@ -208,7 +208,7 @@ class TestRecordTool:time.sleep(0.1)# Assert- assert len(mock_req.request_history) == 4+ assert len(mock_req.request_history) == 5request_json = mock_req.last_request.json()assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keydiff --git a/tests/test_session.py b/tests/test_session.pyindex 075aca84..b40fd04e 100644--- a/tests/test_session.py+++ b/tests/test_session.py@@ -7,17 +7,17 @@ from agentops.singleton import clear_singletons@pytest.fixture(autouse=True)-def setup_teardown():+def setup_teardown(mock_req):clear_singletons()yieldagentops.end_all_sessions() # teardown part[email protected](autouse=True, scope="function")def mock_req():with requests_mock.Mocker() as m:url = "https://api.agentops.ai"- m.post(url + "/v2/create_events", text="ok")+ m.post(url + "/v2/create_events", json={"status": "ok"})m.post(url + "/v2/create_session", json={"status": "success", "jwt": "some_jwt"})@@ -25,7 +25,8 @@ def mock_req():url + "/v2/reauthorize_jwt", json={"status": "success", "jwt": "some_jwt"})m.post(url + "/v2/update_session", json={"status": "success", "token_cost": 5})- m.post(url + "/v2/developer_errors", text="ok")+ m.post(url + "/v2/developer_errors", json={"status": "ok"})+ m.post("https://pypi.org/pypi/agentops/json", status_code=404)yield m@@ -52,12 +53,11 @@ class TestSingleSessions:agentops.record(ActionEvent(self.event_type))agentops.record(ActionEvent(self.event_type))- # We should have 1 requests (session start).- assert len(mock_req.request_history) == 1+ time.sleep(0.1)+ # 3 Requests: check_for_updates, start_session, create_events (2 in 1)+ assert len(mock_req.request_history) == 3time.sleep(0.15)- # We should have 2 requests (session and 2 events combined into 1)- assert len(mock_req.request_history) == 2assert mock_req.last_request.headers["Authorization"] == f"Bearer some_jwt"request_json = mock_req.last_request.json()assert request_json["events"][0]["event_type"] == self.event_type@@ -66,8 +66,8 @@ class TestSingleSessions:agentops.end_session(end_state)time.sleep(0.15)- # We should have 3 requests (additional end session)- assert len(mock_req.request_history) == 3+ # We should have 4 requests (additional end session)+ assert len(mock_req.request_history) == 4assert mock_req.last_request.headers["Authorization"] == f"Bearer some_jwt"request_json = mock_req.last_request.json()assert request_json["session"]["end_state"] == end_state@@ -108,8 +108,8 @@ class TestSingleSessions:agentops.end_session(end_state)time.sleep(0.15)- # Assert 3 requests, 1 for session init, 1 for event, 1 for end session- assert len(mock_req.request_history) == 3+ # 4 requests: check_for_updates, start_session, record_event, end_session+ assert len(mock_req.request_history) == 4assert mock_req.last_request.headers["X-Agentops-Api-Key"] == self.api_keyrequest_json = mock_req.last_request.json()assert request_json["session"]["end_state"] == end_state@@ -192,7 +192,7 @@ class TestMultiSessions:def setup_method(self):self.api_key = "11111111-1111-4111-8111-111111111111"self.event_type = "test_event_type"- agentops.init(api_key=self.api_key, max_wait_time=50, auto_start_session=False)+ agentops.init(api_key=self.api_key, max_wait_time=500, auto_start_session=False)def test_two_sessions(self, mock_req):session_1 = agentops.start_session()@@ -205,17 +205,18 @@ class TestMultiSessions:str(session_1.session_id),str(session_2.session_id),]+ time.sleep(0.1)- # We should have 2 requests (session starts).- assert len(mock_req.request_history) == 2+ # Requests: check_for_updates, 2 start_session+ assert len(mock_req.request_history) == 3session_1.record(ActionEvent(self.event_type))session_2.record(ActionEvent(self.event_type))time.sleep(1.5)- # We should have 4 requests (2 sessions and 2 events each in their own request)- assert len(mock_req.request_history) == 4+ # 5 requests: check_for_updates, 2 start_session, 2 record_event+ assert len(mock_req.request_history) == 5assert mock_req.last_request.headers["Authorization"] == f"Bearer some_jwt"request_json = mock_req.last_request.json()assert request_json["events"][0]["event_type"] == self.event_type@@ -225,16 +226,16 @@ class TestMultiSessions:session_1.end_session(end_state)time.sleep(1.5)- # We should have 6 requests (2 additional end sessions)- assert len(mock_req.request_history) == 5+ # Additional end session request+ assert len(mock_req.request_history) == 6assert mock_req.last_request.headers["Authorization"] == f"Bearer some_jwt"request_json = mock_req.last_request.json()assert request_json["session"]["end_state"] == end_stateassert len(request_json["session"]["tags"]) == 0session_2.end_session(end_state)- # We should have 6 requests (2 additional end sessions)- assert len(mock_req.request_history) == 6+ # Additional end session request+ assert len(mock_req.request_history) == 7assert mock_req.last_request.headers["Authorization"] == f"Bearer some_jwt"request_json = mock_req.last_request.json()assert request_json["session"]["end_state"] == end_statediff --git a/tests/test_teardown.py b/tests/test_teardown.pyindex b86ea079..3789e303 100644--- a/tests/test_teardown.py+++ b/tests/test_teardown.py@@ -1,28 +1,28 @@import pytestimport requests_mock-from agentops import Client+import agentops[email protected](autouse=True, scope="function")def mock_req():with requests_mock.Mocker() as m:url = "https://api.agentops.ai"- m.post(url + "/v2/create_events", text="ok")+ m.post(url + "/v2/create_events", json={"status": "ok"})m.post(url + "/v2/create_session", json={"status": "success", "jwt": "some_jwt"})+ m.post(+ url + "/v2/reauthorize_jwt", json={"status": "success", "jwt": "some_jwt"}+ )m.post(url + "/v2/update_session", json={"status": "success", "token_cost": 5})- m.post(url + "/v2/developer_errors", text="ok")+ m.post(url + "/v2/developer_errors", json={"status": "ok"})+ m.post("https://pypi.org/pypi/agentops/json", status_code=404)yield mclass TestSessions:- def setup_method(self):- self.api_key = "11111111-1111-4111-8111-111111111111"- self.event_type = "test_event_type"- Client().configure(api_key=self.api_key)- Client().initialize()-- def test_exit(self):- # Tests should not hang.- ...+ def test_exit(self, mock_req):+ url = "https://api.agentops.ai"+ api_key = "11111111-1111-4111-8111-111111111111"+ tool_name = "test_tool_name"+ agentops.init(api_key, max_wait_time=5, auto_start_session=False)diff --git a/tox.ini b/tox.iniindex 1e9d06b7..f69422fb 100644--- a/tox.ini+++ b/tox.ini@@ -4,7 +4,7 @@# and then run "tox" from this directory.[tox]-envlist = py37,py38,py39,py310,py311,py312+envlist = py311[testenv]deps =@@ -20,11 +20,12 @@ deps =langchain-corelangchaintermcolor+ -e .commands =- coverage run --source agentops -m pytest+ coverage run --source . -m pytestcoverage report -mcoverage xml- mypy: mypy agentops+ mypy: mypy .[coverage:run]branch = True