Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
import pytest
from unittest.mock import patch, MagicMock
# Tests for the new session management functionality
# These tests call the actual public API but mock the underlying implementation
# to avoid making real API calls or initializing the full telemetry pipeline
@pytest.fixture(scope="function")
def mock_tracing_core():
"""Mock the global tracer to avoid actual initialization"""
# Patch both the main location and where it's imported in client
with (
patch("agentops.tracer") as mock_tracer,
patch("agentops.client.client.tracer", mock_tracer),
patch("agentops.sdk.decorators.factory.tracer", mock_tracer),
patch("agentops.legacy.tracer", mock_tracer),
):
# Create a mock instance
mock_tracer.initialized = True
yield mock_tracer
@pytest.fixture(scope="function")
def mock_api_client():
"""Mock the API client to avoid actual API calls"""
with patch("agentops.client.api.ApiClient") as mock_api:
# Configure the v3.fetch_auth_token method to return a valid response
mock_v3 = MagicMock()
mock_v3.fetch_auth_token.return_value = {"token": "mock-jwt-token", "project_id": "mock-project-id"}
mock_api.return_value.v3 = mock_v3
yield mock_api
@pytest.fixture(scope="function")
def mock_trace_context():
"""Mock the TraceContext creation"""
mock_span = MagicMock()
mock_token = MagicMock()
mock_trace_context_instance = MagicMock()
mock_trace_context_instance.span = mock_span
mock_trace_context_instance.token = mock_token
mock_trace_context_instance.is_init_trace = False
return mock_trace_context_instance
@pytest.fixture(scope="function")
def reset_client():
"""Reset the AgentOps client before each test"""
import agentops
# Create a fresh client instance for each test
agentops._client = agentops.Client()
# Reset all client state
agentops._client._initialized = False
agentops._client._init_trace_context = None
agentops._client._legacy_session_for_init_trace = None
yield
# Clean up after test
try:
if hasattr(agentops._client, "_initialized"):
agentops._client._initialized = False
if hasattr(agentops._client, "_init_trace_context"):
agentops._client._init_trace_context = None
if hasattr(agentops._client, "_legacy_session_for_init_trace"):
agentops._client._legacy_session_for_init_trace = None
except:
pass
def test_explicit_init_then_explicit_start_trace(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test explicitly initializing followed by explicitly starting a trace"""
import agentops
# Explicitly initialize with auto_start_session=False
agentops.init(api_key="test-api-key", auto_start_session=False)
# Verify that no auto-trace was started
mock_tracing_core.start_trace.assert_not_called()
# Mock the start_trace method to return our mock trace context
mock_tracing_core.start_trace.return_value = mock_trace_context
# Explicitly start a trace
trace_context = agentops.start_trace(trace_name="test_trace", tags=["test"])
# Verify the trace was created
mock_tracing_core.start_trace.assert_called_once_with(trace_name="test_trace", tags=["test"])
assert trace_context == mock_trace_context
def test_auto_start_session_true(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test initializing with auto_start_session=True"""
import agentops
from agentops.legacy import Session
# Mock the start_trace method to return our mock trace context
mock_tracing_core.start_trace.return_value = mock_trace_context
# Initialize with auto_start_session=True
result = agentops.init(api_key="test-api-key", auto_start_session=True)
# Verify a trace was auto-started
mock_tracing_core.start_trace.assert_called_once()
# init() should return a Session object when auto-starting a session
assert isinstance(result, Session)
# Check that the session's trace_context has the expected properties
assert result.trace_context is not None
def test_auto_start_session_default(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test initializing with default auto_start_session behavior"""
import agentops
from agentops.legacy import Session
# Mock the start_trace method to return our mock trace context
mock_tracing_core.start_trace.return_value = mock_trace_context
# Initialize without explicitly setting auto_start_session (defaults to True)
result = agentops.init(api_key="test-api-key")
# Verify that the client was initialized
assert agentops._client.initialized
# Since auto_start_session defaults to True, init() should return a Session object
assert isinstance(result, Session)
# Check that the session's trace_context has the expected properties
assert result.trace_context is not None
def test_start_trace_without_init():
"""Test starting a trace without initialization triggers auto-init"""
import agentops
# Reset client for test
agentops._client = agentops.Client()
# Mock global tracer to be uninitialized initially, then initialized after init
with (
patch("agentops.tracer") as mock_tracer,
patch("agentops.client.client.tracer", mock_tracer),
patch("agentops.sdk.decorators.factory.tracer", mock_tracer),
patch("agentops.legacy.tracer", mock_tracer),
):
mock_tracer.initialized = False
# Mock the init function to simulate successful initialization
with patch("agentops.init") as mock_init:
def side_effect():
# After init is called, mark global tracer as initialized
mock_tracer.initialized = True
mock_init.side_effect = side_effect
mock_tracer.start_trace.return_value = None
# Try to start a trace without initialization
result = agentops.start_trace(trace_name="test_trace")
# Verify that init was called automatically
mock_init.assert_called_once()
# Should return None since start_trace returned None
assert result is None
def test_end_trace(mock_tracing_core, mock_trace_context):
"""Test ending a trace"""
import agentops
# End the trace
agentops.end_trace(mock_trace_context, end_state="Success")
# Verify end_trace was called on global tracer
mock_tracing_core.end_trace.assert_called_once_with(trace_context=mock_trace_context, end_state="Success")
def test_session_decorator_creates_trace(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test that the @session decorator creates a trace-level span"""
import agentops
from agentops.sdk.decorators import session
# Initialize AgentOps
agentops.init(api_key="test-api-key", auto_start_session=False)
# Reset the call count to start fresh
mock_tracing_core.reset_mock()
# Mock the start_trace and end_trace methods
mock_tracing_core.start_trace.return_value = mock_trace_context
@session(name="test_session", tags=["decorator_test"])
def test_function():
return "test_result"
# Execute the decorated function
result = test_function()
# Verify the function executed successfully
assert result == "test_result"
# Verify that start_trace and end_trace were called
# Note: The decorator might call start_trace multiple times due to initialization
assert mock_tracing_core.start_trace.call_count >= 1
assert mock_tracing_core.end_trace.call_count >= 1
def test_session_decorator_with_exception(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test that the @session decorator handles exceptions properly"""
import agentops
from agentops.sdk.decorators import session
# Initialize AgentOps
agentops.init(api_key="test-api-key", auto_start_session=False)
# Reset the call count to start fresh
mock_tracing_core.reset_mock()
# Mock the start_trace method
mock_tracing_core.start_trace.return_value = mock_trace_context
@session(name="failing_session")
def failing_function():
raise ValueError("Test exception")
# Execute the decorated function and expect an exception
with pytest.raises(ValueError, match="Test exception"):
failing_function()
# Verify that start_trace was called
assert mock_tracing_core.start_trace.call_count >= 1
# Verify that end_trace was called
assert mock_tracing_core.end_trace.call_count >= 1
def test_legacy_start_session_compatibility(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test that legacy start_session still works and calls tracer.start_trace"""
import agentops
from agentops.legacy import Session
# Initialize AgentOps
agentops.init(api_key="test-api-key", auto_start_session=False)
# Reset the call count to start fresh
mock_tracing_core.reset_mock()
# Mock the start_trace method
mock_tracing_core.start_trace.return_value = mock_trace_context
# Start a legacy session
session = agentops.start_session(tags=["legacy_test"])
# Verify the session was created
assert isinstance(session, Session)
# Check that the session's trace_context has the expected properties
assert session.trace_context is not None
# Verify that tracer.start_trace was called
# Note: May be called multiple times due to initialization
assert mock_tracing_core.start_trace.call_count >= 1
def test_legacy_end_session_compatibility(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test that legacy end_session still works and calls tracer.end_trace"""
import agentops
from agentops.legacy import Session
# Initialize AgentOps
agentops.init(api_key="test-api-key", auto_start_session=False)
# Reset the call count to start fresh
mock_tracing_core.reset_mock()
# Create a legacy session object
session = Session(mock_trace_context)
# End the session
agentops.end_session(session)
# Verify that tracer.end_trace was called
mock_tracing_core.end_trace.assert_called_once_with(mock_trace_context, end_state="Success")
def test_no_double_init(mock_tracing_core, mock_api_client, reset_client):
"""Test that calling init multiple times doesn't reinitialize"""
import agentops
# Initialize once
agentops.init(api_key="test-api-key", auto_start_session=False)
# Track the call count
call_count = mock_api_client.call_count
# Call init again with the same API key
agentops.init(api_key="test-api-key", auto_start_session=False)
# Verify that API client wasn't constructed again
assert mock_api_client.call_count == call_count
def test_client_initialization_behavior(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test basic client initialization behavior"""
import agentops
# Mock the start_trace method
mock_tracing_core.start_trace.return_value = mock_trace_context
# Test that initialization works
agentops.init(api_key="test-api-key", auto_start_session=False)
# Verify that the client was initialized
assert agentops._client.initialized
# The API client might not be called if already mocked at a higher level
# Just verify that the initialization completed successfully
# Test that calling init again doesn't cause issues
agentops.init(api_key="test-api-key", auto_start_session=False)
# Should still be initialized
assert agentops._client.initialized
def test_multiple_concurrent_traces(mock_tracing_core, mock_api_client, reset_client):
"""Test that multiple traces can be started concurrently"""
import agentops
# Initialize AgentOps
agentops.init(api_key="test-api-key", auto_start_session=False)
# Create mock trace contexts for different traces
mock_trace_context_1 = MagicMock()
mock_trace_context_2 = MagicMock()
# Mock start_trace to return different contexts
mock_tracing_core.start_trace.side_effect = [
mock_trace_context_1,
mock_trace_context_2,
]
# Start multiple traces
trace1 = agentops.start_trace(trace_name="trace1", tags=["test1"])
trace2 = agentops.start_trace(trace_name="trace2", tags=["test2"])
# Verify both traces were created
assert trace1 == mock_trace_context_1
assert trace2 == mock_trace_context_2
# Verify start_trace was called twice
assert mock_tracing_core.start_trace.call_count == 2
def test_trace_context_properties(mock_trace_context):
"""Test that TraceContext properties work correctly"""
from agentops.legacy import Session
# Create a legacy session with the mock trace context
session = Session(mock_trace_context)
# Test that properties are accessible
assert session.span == mock_trace_context.span
assert session.token == mock_trace_context.token
assert session.trace_context == mock_trace_context
def test_session_decorator_async_function(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test that the @session decorator works with async functions"""
import agentops
import asyncio
from agentops.sdk.decorators import session
# Initialize AgentOps
agentops.init(api_key="test-api-key", auto_start_session=False)
# Reset the call count to start fresh
mock_tracing_core.reset_mock()
# Mock the start_trace method
mock_tracing_core.start_trace.return_value = mock_trace_context
@session(name="async_test_session")
async def async_test_function():
await asyncio.sleep(0.01) # Simulate async work
return "async_result"
# Execute the decorated async function
result = asyncio.run(async_test_function())
# Verify the function executed successfully
assert result == "async_result"
# Verify that start_trace and end_trace were called
assert mock_tracing_core.start_trace.call_count >= 1
assert mock_tracing_core.end_trace.call_count >= 1
def test_trace_context_creation():
"""Test that TraceContext can be created with proper attributes"""
from agentops.sdk.core import TraceContext
mock_span = MagicMock()
mock_token = MagicMock()
# Test creating a TraceContext
trace_context = TraceContext(span=mock_span, token=mock_token, is_init_trace=False)
assert trace_context.span == mock_span
assert trace_context.token == mock_token
assert trace_context.is_init_trace == False
def test_session_management_integration():
"""Test the integration between new and legacy session management"""
import agentops
# Reset client for test
agentops._client = agentops.Client()
# Test that we can use both new and legacy APIs together
with (
patch("agentops.tracer") as mock_tracer,
patch("agentops.client.client.tracer", mock_tracer),
patch("agentops.sdk.decorators.factory.tracer", mock_tracer),
patch("agentops.legacy.tracer", mock_tracer),
):
mock_tracer.initialized = True
# Mock API client
with patch("agentops.client.api.ApiClient") as mock_api:
mock_v3 = MagicMock()
mock_v3.fetch_auth_token.return_value = {"token": "mock-jwt-token", "project_id": "mock-project-id"}
mock_api.return_value.v3 = mock_v3
# Initialize AgentOps
agentops.init(api_key="test-api-key", auto_start_session=False)
# Reset call counts after initialization
mock_tracer.reset_mock()
# Create mock trace context
mock_trace_context = MagicMock()
mock_tracer.start_trace.return_value = mock_trace_context
# Test new API
trace_context = agentops.start_trace(trace_name="new_api_trace")
assert trace_context == mock_trace_context
# Test legacy API
session = agentops.start_session(tags=["legacy"])
# Check that the session's trace_context has the expected properties
assert session.trace_context is not None
# Test ending both
agentops.end_trace(trace_context)
agentops.end_session(session)
# Verify calls were made
assert mock_tracer.start_trace.call_count >= 2
assert mock_tracer.end_trace.call_count >= 2
# CrewAI Backwards Compatibility Tests
# These tests ensure CrewAI integration patterns continue to work
def test_session_auto_start(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""Test auto-start session functionality for CrewAI compatibility"""
import agentops
from agentops.legacy import Session
# Configure mocks for session initialization
mock_tracing_core.start_trace.return_value = mock_trace_context
# Pass a dummy API key for the test
session = agentops.init(api_key="test-api-key", auto_start_session=True)
assert isinstance(session, Session)
def test_crewai_backwards_compatibility(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""
CrewAI needs to access:
agentops.track_agent
agentops.track_tool
agentops.start_session
agentops.end_session
agentops.ActionEvent
agentops.ToolEvent
"""
import agentops
from agentops.legacy import Session
# Configure mocks
mock_tracing_core.start_trace.return_value = mock_trace_context
# Test initialization with API key
agentops.init(api_key="test-api-key")
# Test session management functions
session = agentops.start_session(tags=["test", "crewai"])
assert isinstance(session, Session)
# Test that passing a string to end_session doesn't raise an error
agentops.end_session("Success") # This pattern is used in CrewAI
# Test track_agent function exists and doesn't raise errors
try:
# Mock an agent object similar to what CrewAI would provide
class MockAgent:
def __init__(self):
self.role = "Test Agent"
self.goal = "Testing"
self.id = "test-agent-id"
agent = MockAgent()
agentops.track_agent(agent)
except Exception as e:
assert False, f"track_agent raised an exception: {e}"
# Test track_tool function exists and doesn't raise errors
try:
# Mock a tool object similar to what CrewAI would provide
class MockTool:
def __init__(self):
self.name = "Test Tool"
self.description = "A test tool"
tool = MockTool()
agentops.track_tool(tool, "Test Agent")
except Exception as e:
assert False, f"track_tool raised an exception: {e}"
# Test events that CrewAI might use
tool_event = agentops.ToolEvent(name="test_tool")
action_event = agentops.ActionEvent(action_type="test_action")
# Verify that record function works with these events
agentops.record(tool_event)
agentops.record(action_event)
def test_crewai_kwargs_pattern(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""
Test the CrewAI < 0.105.0 pattern where end_session is called with only kwargs.
In versions < 0.105.0, CrewAI directly calls:
agentops.end_session(
end_state="Success",
end_state_reason="Finished Execution",
is_auto_end=True
)
"""
import agentops
from agentops.legacy import Session
# Configure mocks
mock_tracing_core.start_trace.return_value = mock_trace_context
# Initialize with test API key
agentops.init(api_key="test-api-key")
# Create a session
session = agentops.start_session(tags=["test", "crewai-kwargs"])
assert isinstance(session, Session)
# Test the CrewAI < 0.105.0 pattern - calling end_session with only kwargs
agentops.end_session(end_state="Success", end_state_reason="Finished Execution", is_auto_end=True)
# After calling end_session, creating a new session should work correctly
# (this implicitly tests that the internal state is reset properly)
new_session = agentops.start_session(tags=["test", "post-end"])
assert isinstance(new_session, Session)
def test_crewai_kwargs_pattern_no_session(mock_tracing_core, mock_api_client, reset_client):
"""
Test the CrewAI < 0.105.0 pattern where end_session is called with only kwargs,
but no session has been created.
This should log a warning but not fail.
"""
import agentops
# Initialize with test API key
agentops.init(api_key="test-api-key")
# We don't need to explicitly clear the session state
# Just make sure we start with a clean state by calling init
# Test the CrewAI < 0.105.0 pattern - calling end_session with only kwargs
# when no session exists. This should not raise an error.
agentops.end_session(end_state="Success", end_state_reason="Finished Execution", is_auto_end=True)
def test_crewai_kwargs_force_flush(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""
Test that when using the CrewAI < 0.105.0 pattern (end_session with kwargs),
the spans are properly exported to the backend with force_flush.
This is a more comprehensive test that ensures spans are actually sent
to the backend when using the CrewAI integration pattern.
"""
import agentops
import time
# Configure mocks
mock_tracing_core.start_trace.return_value = mock_trace_context
# Initialize AgentOps with API key
agentops.init(api_key="test-api-key")
# Create a session
agentops.start_session(tags=["test", "crewai-integration"])
# Simulate some work
time.sleep(0.1)
# End session with kwargs (CrewAI < 0.105.0 pattern)
agentops.end_session(end_state="Success", end_state_reason="Test Finished", is_auto_end=True)
# Explicitly ensure the core isn't already shut down for the test
# Note: We're using mocks, so we check the mock's initialized state
assert mock_tracing_core.initialized, "Mocked tracer should still be initialized"
def test_crewai_task_instrumentation(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
"""
Test the CrewAI task instrumentation focusing on span attributes and tags.
This test verifies that task spans are properly created with correct attributes
and tags without requiring a session.
"""
import agentops
from opentelemetry.trace import SpanKind
from agentops.semconv import SpanAttributes, AgentOpsSpanKindValues
from opentelemetry import trace
from agentops.semconv.core import CoreAttributes
# Configure mocks
mock_tracing_core.start_trace.return_value = mock_trace_context
# Initialize AgentOps with API key and default tags
agentops.init(
api_key="test-api-key",
)
agentops.start_session(tags=["test", "crewai-integration"])
# Get the tracer
tracer = trace.get_tracer(__name__)
# Create a mock task instance
class MockTask:
def __init__(self):
self.description = "Test Task Description"
self.agent = "Test Agent"
self.tools = ["tool1", "tool2"]
task = MockTask()
# Start a span for the task
with tracer.start_as_current_span(
f"{task.description}.task",
kind=SpanKind.CLIENT,
attributes={
SpanAttributes.AGENTOPS_SPAN_KIND: AgentOpsSpanKindValues.TASK.value,
CoreAttributes.TAGS: ["crewai", "task-test"],
},
) as span:
# Verify span attributes
assert span.attributes[SpanAttributes.AGENTOPS_SPAN_KIND] == AgentOpsSpanKindValues.TASK.value
assert "crewai" in span.attributes[CoreAttributes.TAGS]
assert "task-test" in span.attributes[CoreAttributes.TAGS]
# Verify span name
assert span.name == f"{task.description}.task"
# Verify span kind
assert span.kind == SpanKind.CLIENT
agentops.end_session(end_state="Success", end_state_reason="Test Finished", is_auto_end=True)