Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

6.8 KB
Raw Blame
from typing import Any, Dict, List, Protocol, Tuple, Union
import importlib
import unittest.mock as mock

from opentelemetry import trace as trace_api
from opentelemetry.sdk.trace import ReadableSpan, Span, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.util.types import Attributes

from agentops.sdk.core import tracer


def create_tracer_provider(
    **kwargs,
) -> Tuple[TracerProvider, InMemorySpanExporter, SimpleSpanProcessor]:
    """Helper to create a configured tracer provider.

    Creates and configures a `TracerProvider` with a
    `SimpleSpanProcessor` and a `InMemorySpanExporter`.
    All the parameters passed are forwarded to the TracerProvider
    constructor.

    Returns:
        A tuple with the tracer provider, memory exporter, and span processor.
    """
    tracer_provider = TracerProvider(**kwargs)
    memory_exporter = InMemorySpanExporter()

    # Use SimpleSpanProcessor instead of both processors to avoid duplication
    span_processor = SimpleSpanProcessor(memory_exporter)
    tracer_provider.add_span_processor(span_processor)

    return tracer_provider, memory_exporter, span_processor


def reset_trace_globals():
    """Reset the global trace state to avoid conflicts."""
    # Reset tracer provider
    trace_api._TRACER_PROVIDER = None

    # Reload the trace module to clear warning state
    importlib.reload(trace_api)


class HasAttributesViaProperty(Protocol):
    @property
    def attributes(self) -> Attributes: ...


class HasAttributesViaAttr(Protocol):
    attributes: Attributes


HasAttributes = Union[HasAttributesViaProperty, HasAttributesViaAttr]


class InstrumentationTester:
    """
    A utility class for testing instrumentation in the AgentOps SDK.

    This class provides methods for setting up a test environment with
    in-memory span exporters, and for asserting properties of spans.
    """

    tracer_provider: TracerProvider
    memory_exporter: InMemorySpanExporter
    span_processor: SimpleSpanProcessor

    def __init__(self):
        """Initialize the instrumentation tester."""
        # Reset any global state first
        reset_trace_globals()

        # Shut down any existing tracing core
        # self._shutdown_core()

        # Create a new tracer provider and memory exporter
        (
            self.tracer_provider,
            self.memory_exporter,
            self.span_processor,
        ) = create_tracer_provider()

        # Set the tracer provider
        trace_api.set_tracer_provider(self.tracer_provider)

        # Create a mock for the meter provider
        self.mock_meter_provider = mock.MagicMock()

        # Patch the setup_telemetry function to return our test providers
        self.setup_telemetry_patcher = mock.patch(
            "agentops.sdk.core.setup_telemetry", return_value=(self.tracer_provider, self.mock_meter_provider)
        )
        self.mock_setup_telemetry = self.setup_telemetry_patcher.start()

        # Reset the tracing core to force reinitialization
        core = tracer
        core._initialized = False
        core._provider = None

        # Initialize the core, which will now use our mocked setup_telemetry
        core.initialize()

        self.clear_spans()

    def _shutdown_core(self):
        """Safely shut down the tracing core."""
        try:
            tracer.shutdown()
        except Exception as e:
            print(f"Warning: Error shutting down tracing core: {e}")

    def clear_spans(self):
        """Clear all spans from the memory exporter."""
        # Force flush spans
        self.span_processor.force_flush()

        # Then clear the memory
        self.memory_exporter.clear()
        print("Cleared all spans from memory exporter")

    def reset(self):
        """Reset the instrumentation tester."""
        # Force flush any pending spans
        self.span_processor.force_flush()

        # Clear any existing spans
        self.clear_spans()

        # Reset global trace state
        reset_trace_globals()

        # Set our tracer provider again
        trace_api.set_tracer_provider(self.tracer_provider)

        # Shut down and re-initialize the tracing core
        self._shutdown_core()

        # Reset the mock setup_telemetry function
        self.mock_setup_telemetry.reset_mock()

        # Reset the tracing core to force reinitialization
        core = tracer
        core._initialized = False
        core._provider = None

        # Initialize the core, which will now use our mocked setup_telemetry
        core.initialize()

    def __del__(self):
        """Clean up resources when the tester is garbage collected."""
        try:
            # Stop the patcher when the tester is deleted
            self.setup_telemetry_patcher.stop()
        except Exception:
            pass

    def get_finished_spans(self) -> List[ReadableSpan]:
        """Get all finished spans."""
        # Force flush any pending spans
        self.span_processor.force_flush()

        # Get the spans
        spans = list(self.memory_exporter.get_finished_spans())
        print(f"Instrumentation Tester: Found {len(spans)} finished spans")
        for i, span in enumerate(spans):
            print(f"Span {i}: name={span.name}, attributes={span.attributes}")
        return spans

    def get_spans_by_name(self, name: str) -> List[ReadableSpan]:
        """Get all spans with the given name."""
        return [span for span in self.get_finished_spans() if span.name == name]

    def get_spans_by_kind(self, kind: str) -> List[ReadableSpan]:
        """Get all spans with the given kind."""
        return [
            span for span in self.get_finished_spans() if span.attributes and span.attributes.get("span.kind") == kind
        ]

    @staticmethod
    def assert_has_attributes(obj: HasAttributes, attributes: Dict[str, Any]):
        """Assert that an object has the given attributes."""
        import json

        assert obj.attributes is not None
        for key, val in attributes.items():
            assert key in obj.attributes, f"Key {key!r} not found in attributes"

            actual_val = obj.attributes[key]

            # Try to handle JSON-serialized values
            if isinstance(actual_val, str) and isinstance(val, (list, dict)):
                try:
                    actual_val = json.loads(actual_val)
                except json.JSONDecodeError:
                    pass

            assert actual_val == val, f"Value for key {key!r} does not match: {actual_val} != {val}"

    @staticmethod
    def assert_span_instrumented_for(span: Union[Span, ReadableSpan], module):
        """Assert that a span is instrumented for the given module."""
        assert span.instrumentation_scope is not None
        assert span.instrumentation_scope.name == module.__name__
        assert span.instrumentation_scope.version == module.__version__