Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

23.7 KB
Raw Blame
import pytest
from unittest.mock import patch, MagicMock

# Tests for the new session management functionality
# These tests call the actual public API but mock the underlying implementation
# to avoid making real API calls or initializing the full telemetry pipeline


@pytest.fixture(scope="function")
def mock_tracing_core():
    """Mock the global tracer to avoid actual initialization"""
    # Patch both the main location and where it's imported in client
    with (
        patch("agentops.tracer") as mock_tracer,
        patch("agentops.client.client.tracer", mock_tracer),
        patch("agentops.sdk.decorators.factory.tracer", mock_tracer),
        patch("agentops.legacy.tracer", mock_tracer),
    ):
        # Create a mock instance
        mock_tracer.initialized = True

        yield mock_tracer


@pytest.fixture(scope="function")
def mock_api_client():
    """Mock the API client to avoid actual API calls"""
    with patch("agentops.client.api.ApiClient") as mock_api:
        # Configure the v3.fetch_auth_token method to return a valid response
        mock_v3 = MagicMock()
        mock_v3.fetch_auth_token.return_value = {"token": "mock-jwt-token", "project_id": "mock-project-id"}
        mock_api.return_value.v3 = mock_v3

        yield mock_api


@pytest.fixture(scope="function")
def mock_trace_context():
    """Mock the TraceContext creation"""
    mock_span = MagicMock()
    mock_token = MagicMock()
    mock_trace_context_instance = MagicMock()
    mock_trace_context_instance.span = mock_span
    mock_trace_context_instance.token = mock_token
    mock_trace_context_instance.is_init_trace = False

    return mock_trace_context_instance


@pytest.fixture(scope="function")
def reset_client():
    """Reset the AgentOps client before each test"""
    import agentops

    # Create a fresh client instance for each test
    agentops._client = agentops.Client()
    # Reset all client state
    agentops._client._initialized = False
    agentops._client._init_trace_context = None
    agentops._client._legacy_session_for_init_trace = None
    yield
    # Clean up after test
    try:
        if hasattr(agentops._client, "_initialized"):
            agentops._client._initialized = False
        if hasattr(agentops._client, "_init_trace_context"):
            agentops._client._init_trace_context = None
        if hasattr(agentops._client, "_legacy_session_for_init_trace"):
            agentops._client._legacy_session_for_init_trace = None
    except:
        pass


def test_explicit_init_then_explicit_start_trace(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test explicitly initializing followed by explicitly starting a trace"""
    import agentops

    # Explicitly initialize with auto_start_session=False
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Verify that no auto-trace was started
    mock_tracing_core.start_trace.assert_not_called()

    # Mock the start_trace method to return our mock trace context
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Explicitly start a trace
    trace_context = agentops.start_trace(trace_name="test_trace", tags=["test"])

    # Verify the trace was created
    mock_tracing_core.start_trace.assert_called_once_with(trace_name="test_trace", tags=["test"])
    assert trace_context == mock_trace_context


def test_auto_start_session_true(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test initializing with auto_start_session=True"""
    import agentops
    from agentops.legacy import Session

    # Mock the start_trace method to return our mock trace context
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Initialize with auto_start_session=True
    result = agentops.init(api_key="test-api-key", auto_start_session=True)

    # Verify a trace was auto-started
    mock_tracing_core.start_trace.assert_called_once()
    # init() should return a Session object when auto-starting a session
    assert isinstance(result, Session)
    # Check that the session's trace_context has the expected properties
    assert result.trace_context is not None


def test_auto_start_session_default(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test initializing with default auto_start_session behavior"""
    import agentops
    from agentops.legacy import Session

    # Mock the start_trace method to return our mock trace context
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Initialize without explicitly setting auto_start_session (defaults to True)
    result = agentops.init(api_key="test-api-key")

    # Verify that the client was initialized
    assert agentops._client.initialized
    # Since auto_start_session defaults to True, init() should return a Session object
    assert isinstance(result, Session)
    # Check that the session's trace_context has the expected properties
    assert result.trace_context is not None


def test_start_trace_without_init():
    """Test starting a trace without initialization triggers auto-init"""
    import agentops

    # Reset client for test
    agentops._client = agentops.Client()

    # Mock global tracer to be uninitialized initially, then initialized after init
    with (
        patch("agentops.tracer") as mock_tracer,
        patch("agentops.client.client.tracer", mock_tracer),
        patch("agentops.sdk.decorators.factory.tracer", mock_tracer),
        patch("agentops.legacy.tracer", mock_tracer),
    ):
        mock_tracer.initialized = False

        # Mock the init function to simulate successful initialization
        with patch("agentops.init") as mock_init:

            def side_effect():
                # After init is called, mark global tracer as initialized
                mock_tracer.initialized = True

            mock_init.side_effect = side_effect
            mock_tracer.start_trace.return_value = None

            # Try to start a trace without initialization
            result = agentops.start_trace(trace_name="test_trace")

            # Verify that init was called automatically
            mock_init.assert_called_once()
            # Should return None since start_trace returned None
            assert result is None


def test_end_trace(mock_tracing_core, mock_trace_context):
    """Test ending a trace"""
    import agentops

    # End the trace
    agentops.end_trace(mock_trace_context, end_state="Success")

    # Verify end_trace was called on global tracer
    mock_tracing_core.end_trace.assert_called_once_with(trace_context=mock_trace_context, end_state="Success")


def test_session_decorator_creates_trace(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test that the @session decorator creates a trace-level span"""
    import agentops
    from agentops.sdk.decorators import session

    # Initialize AgentOps
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Reset the call count to start fresh
    mock_tracing_core.reset_mock()
    # Mock the start_trace and end_trace methods
    mock_tracing_core.start_trace.return_value = mock_trace_context

    @session(name="test_session", tags=["decorator_test"])
    def test_function():
        return "test_result"

    # Execute the decorated function
    result = test_function()

    # Verify the function executed successfully
    assert result == "test_result"

    # Verify that start_trace and end_trace were called
    # Note: The decorator might call start_trace multiple times due to initialization
    assert mock_tracing_core.start_trace.call_count >= 1
    assert mock_tracing_core.end_trace.call_count >= 1


def test_session_decorator_with_exception(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test that the @session decorator handles exceptions properly"""
    import agentops
    from agentops.sdk.decorators import session

    # Initialize AgentOps
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Reset the call count to start fresh
    mock_tracing_core.reset_mock()
    # Mock the start_trace method
    mock_tracing_core.start_trace.return_value = mock_trace_context

    @session(name="failing_session")
    def failing_function():
        raise ValueError("Test exception")

    # Execute the decorated function and expect an exception
    with pytest.raises(ValueError, match="Test exception"):
        failing_function()

    # Verify that start_trace was called
    assert mock_tracing_core.start_trace.call_count >= 1
    # Verify that end_trace was called
    assert mock_tracing_core.end_trace.call_count >= 1


def test_legacy_start_session_compatibility(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test that legacy start_session still works and calls tracer.start_trace"""
    import agentops
    from agentops.legacy import Session

    # Initialize AgentOps
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Reset the call count to start fresh
    mock_tracing_core.reset_mock()
    # Mock the start_trace method
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Start a legacy session
    session = agentops.start_session(tags=["legacy_test"])

    # Verify the session was created
    assert isinstance(session, Session)
    # Check that the session's trace_context has the expected properties
    assert session.trace_context is not None

    # Verify that tracer.start_trace was called
    # Note: May be called multiple times due to initialization
    assert mock_tracing_core.start_trace.call_count >= 1


def test_legacy_end_session_compatibility(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test that legacy end_session still works and calls tracer.end_trace"""
    import agentops
    from agentops.legacy import Session

    # Initialize AgentOps
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Reset the call count to start fresh
    mock_tracing_core.reset_mock()

    # Create a legacy session object
    session = Session(mock_trace_context)

    # End the session
    agentops.end_session(session)

    # Verify that tracer.end_trace was called
    mock_tracing_core.end_trace.assert_called_once_with(mock_trace_context, end_state="Success")


def test_no_double_init(mock_tracing_core, mock_api_client, reset_client):
    """Test that calling init multiple times doesn't reinitialize"""
    import agentops

    # Initialize once
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Track the call count
    call_count = mock_api_client.call_count

    # Call init again with the same API key
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Verify that API client wasn't constructed again
    assert mock_api_client.call_count == call_count


def test_client_initialization_behavior(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test basic client initialization behavior"""
    import agentops

    # Mock the start_trace method
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Test that initialization works
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Verify that the client was initialized
    assert agentops._client.initialized

    # The API client might not be called if already mocked at a higher level
    # Just verify that the initialization completed successfully

    # Test that calling init again doesn't cause issues
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Should still be initialized
    assert agentops._client.initialized


def test_multiple_concurrent_traces(mock_tracing_core, mock_api_client, reset_client):
    """Test that multiple traces can be started concurrently"""
    import agentops

    # Initialize AgentOps
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Create mock trace contexts for different traces
    mock_trace_context_1 = MagicMock()
    mock_trace_context_2 = MagicMock()

    # Mock start_trace to return different contexts
    mock_tracing_core.start_trace.side_effect = [
        mock_trace_context_1,
        mock_trace_context_2,
    ]

    # Start multiple traces
    trace1 = agentops.start_trace(trace_name="trace1", tags=["test1"])
    trace2 = agentops.start_trace(trace_name="trace2", tags=["test2"])

    # Verify both traces were created
    assert trace1 == mock_trace_context_1
    assert trace2 == mock_trace_context_2

    # Verify start_trace was called twice
    assert mock_tracing_core.start_trace.call_count == 2


def test_trace_context_properties(mock_trace_context):
    """Test that TraceContext properties work correctly"""
    from agentops.legacy import Session

    # Create a legacy session with the mock trace context
    session = Session(mock_trace_context)

    # Test that properties are accessible
    assert session.span == mock_trace_context.span
    assert session.token == mock_trace_context.token
    assert session.trace_context == mock_trace_context


def test_session_decorator_async_function(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test that the @session decorator works with async functions"""
    import agentops
    import asyncio
    from agentops.sdk.decorators import session

    # Initialize AgentOps
    agentops.init(api_key="test-api-key", auto_start_session=False)

    # Reset the call count to start fresh
    mock_tracing_core.reset_mock()
    # Mock the start_trace method
    mock_tracing_core.start_trace.return_value = mock_trace_context

    @session(name="async_test_session")
    async def async_test_function():
        await asyncio.sleep(0.01)  # Simulate async work
        return "async_result"

    # Execute the decorated async function
    result = asyncio.run(async_test_function())

    # Verify the function executed successfully
    assert result == "async_result"

    # Verify that start_trace and end_trace were called
    assert mock_tracing_core.start_trace.call_count >= 1
    assert mock_tracing_core.end_trace.call_count >= 1


def test_trace_context_creation():
    """Test that TraceContext can be created with proper attributes"""
    from agentops.sdk.core import TraceContext

    mock_span = MagicMock()
    mock_token = MagicMock()

    # Test creating a TraceContext
    trace_context = TraceContext(span=mock_span, token=mock_token, is_init_trace=False)

    assert trace_context.span == mock_span
    assert trace_context.token == mock_token
    assert trace_context.is_init_trace == False


def test_session_management_integration():
    """Test the integration between new and legacy session management"""
    import agentops

    # Reset client for test
    agentops._client = agentops.Client()

    # Test that we can use both new and legacy APIs together
    with (
        patch("agentops.tracer") as mock_tracer,
        patch("agentops.client.client.tracer", mock_tracer),
        patch("agentops.sdk.decorators.factory.tracer", mock_tracer),
        patch("agentops.legacy.tracer", mock_tracer),
    ):
        mock_tracer.initialized = True

        # Mock API client
        with patch("agentops.client.api.ApiClient") as mock_api:
            mock_v3 = MagicMock()
            mock_v3.fetch_auth_token.return_value = {"token": "mock-jwt-token", "project_id": "mock-project-id"}
            mock_api.return_value.v3 = mock_v3

            # Initialize AgentOps
            agentops.init(api_key="test-api-key", auto_start_session=False)

            # Reset call counts after initialization
            mock_tracer.reset_mock()

            # Create mock trace context
            mock_trace_context = MagicMock()
            mock_tracer.start_trace.return_value = mock_trace_context

            # Test new API
            trace_context = agentops.start_trace(trace_name="new_api_trace")
            assert trace_context == mock_trace_context

            # Test legacy API
            session = agentops.start_session(tags=["legacy"])
            # Check that the session's trace_context has the expected properties
            assert session.trace_context is not None

            # Test ending both
            agentops.end_trace(trace_context)
            agentops.end_session(session)

            # Verify calls were made
            assert mock_tracer.start_trace.call_count >= 2
            assert mock_tracer.end_trace.call_count >= 2


# CrewAI Backwards Compatibility Tests
# These tests ensure CrewAI integration patterns continue to work


def test_session_auto_start(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """Test auto-start session functionality for CrewAI compatibility"""
    import agentops
    from agentops.legacy import Session

    # Configure mocks for session initialization
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Pass a dummy API key for the test
    session = agentops.init(api_key="test-api-key", auto_start_session=True)

    assert isinstance(session, Session)


def test_crewai_backwards_compatibility(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """
    CrewAI needs to access:

    agentops.track_agent
    agentops.track_tool
    agentops.start_session
    agentops.end_session
    agentops.ActionEvent
    agentops.ToolEvent
    """
    import agentops
    from agentops.legacy import Session

    # Configure mocks
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Test initialization with API key
    agentops.init(api_key="test-api-key")

    # Test session management functions
    session = agentops.start_session(tags=["test", "crewai"])
    assert isinstance(session, Session)

    # Test that passing a string to end_session doesn't raise an error
    agentops.end_session("Success")  # This pattern is used in CrewAI

    # Test track_agent function exists and doesn't raise errors
    try:
        # Mock an agent object similar to what CrewAI would provide
        class MockAgent:
            def __init__(self):
                self.role = "Test Agent"
                self.goal = "Testing"
                self.id = "test-agent-id"

        agent = MockAgent()
        agentops.track_agent(agent)
    except Exception as e:
        assert False, f"track_agent raised an exception: {e}"

    # Test track_tool function exists and doesn't raise errors
    try:
        # Mock a tool object similar to what CrewAI would provide
        class MockTool:
            def __init__(self):
                self.name = "Test Tool"
                self.description = "A test tool"

        tool = MockTool()
        agentops.track_tool(tool, "Test Agent")
    except Exception as e:
        assert False, f"track_tool raised an exception: {e}"

    # Test events that CrewAI might use
    tool_event = agentops.ToolEvent(name="test_tool")
    action_event = agentops.ActionEvent(action_type="test_action")

    # Verify that record function works with these events
    agentops.record(tool_event)
    agentops.record(action_event)


def test_crewai_kwargs_pattern(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """
    Test the CrewAI < 0.105.0 pattern where end_session is called with only kwargs.

    In versions < 0.105.0, CrewAI directly calls:
    agentops.end_session(
        end_state="Success",
        end_state_reason="Finished Execution",
        is_auto_end=True
    )
    """
    import agentops
    from agentops.legacy import Session

    # Configure mocks
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Initialize with test API key
    agentops.init(api_key="test-api-key")

    # Create a session
    session = agentops.start_session(tags=["test", "crewai-kwargs"])
    assert isinstance(session, Session)

    # Test the CrewAI < 0.105.0 pattern - calling end_session with only kwargs
    agentops.end_session(end_state="Success", end_state_reason="Finished Execution", is_auto_end=True)

    # After calling end_session, creating a new session should work correctly
    # (this implicitly tests that the internal state is reset properly)
    new_session = agentops.start_session(tags=["test", "post-end"])
    assert isinstance(new_session, Session)


def test_crewai_kwargs_pattern_no_session(mock_tracing_core, mock_api_client, reset_client):
    """
    Test the CrewAI < 0.105.0 pattern where end_session is called with only kwargs,
    but no session has been created.

    This should log a warning but not fail.
    """
    import agentops

    # Initialize with test API key
    agentops.init(api_key="test-api-key")

    # We don't need to explicitly clear the session state
    # Just make sure we start with a clean state by calling init

    # Test the CrewAI < 0.105.0 pattern - calling end_session with only kwargs
    # when no session exists. This should not raise an error.
    agentops.end_session(end_state="Success", end_state_reason="Finished Execution", is_auto_end=True)


def test_crewai_kwargs_force_flush(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """
    Test that when using the CrewAI < 0.105.0 pattern (end_session with kwargs),
    the spans are properly exported to the backend with force_flush.

    This is a more comprehensive test that ensures spans are actually sent
    to the backend when using the CrewAI integration pattern.
    """
    import agentops
    import time

    # Configure mocks
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Initialize AgentOps with API key
    agentops.init(api_key="test-api-key")

    # Create a session
    agentops.start_session(tags=["test", "crewai-integration"])

    # Simulate some work
    time.sleep(0.1)

    # End session with kwargs (CrewAI < 0.105.0 pattern)
    agentops.end_session(end_state="Success", end_state_reason="Test Finished", is_auto_end=True)

    # Explicitly ensure the core isn't already shut down for the test
    # Note: We're using mocks, so we check the mock's initialized state
    assert mock_tracing_core.initialized, "Mocked tracer should still be initialized"


def test_crewai_task_instrumentation(mock_tracing_core, mock_api_client, mock_trace_context, reset_client):
    """
    Test the CrewAI task instrumentation focusing on span attributes and tags.
    This test verifies that task spans are properly created with correct attributes
    and tags without requiring a session.
    """
    import agentops
    from opentelemetry.trace import SpanKind
    from agentops.semconv import SpanAttributes, AgentOpsSpanKindValues
    from opentelemetry import trace
    from agentops.semconv.core import CoreAttributes

    # Configure mocks
    mock_tracing_core.start_trace.return_value = mock_trace_context

    # Initialize AgentOps with API key and default tags
    agentops.init(
        api_key="test-api-key",
    )
    agentops.start_session(tags=["test", "crewai-integration"])
    # Get the tracer
    tracer = trace.get_tracer(__name__)

    # Create a mock task instance
    class MockTask:
        def __init__(self):
            self.description = "Test Task Description"
            self.agent = "Test Agent"
            self.tools = ["tool1", "tool2"]

    task = MockTask()

    # Start a span for the task
    with tracer.start_as_current_span(
        f"{task.description}.task",
        kind=SpanKind.CLIENT,
        attributes={
            SpanAttributes.AGENTOPS_SPAN_KIND: AgentOpsSpanKindValues.TASK.value,
            CoreAttributes.TAGS: ["crewai", "task-test"],
        },
    ) as span:
        # Verify span attributes
        assert span.attributes[SpanAttributes.AGENTOPS_SPAN_KIND] == AgentOpsSpanKindValues.TASK.value
        assert "crewai" in span.attributes[CoreAttributes.TAGS]
        assert "task-test" in span.attributes[CoreAttributes.TAGS]

        # Verify span name
        assert span.name == f"{task.description}.task"

        # Verify span kind
        assert span.kind == SpanKind.CLIENT

    agentops.end_session(end_state="Success", end_state_reason="Test Finished", is_auto_end=True)