Skip to content

armgit / burr

Public Imported from GitHub
0

Build applications that make decisions (chatbots, agents, simulations, etc...). Monitor, trace, persist, and execute on your own infrastructure.

13.7 KB
Raw Blame
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import contextvars
import json
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import Mock, patch

import pydantic
import pytest
from opentelemetry import context as otel_context
from opentelemetry.sdk.trace import Span, TracerProvider
from opentelemetry.trace import SpanContext

from burr.core import State, serde
from burr.core.action import Action
from burr.integrations.opentelemetry import (
    BurrTrackingSpanProcessor,
    FullSpanContext,
    OpenTelemetryTracker,
    convert_to_otel_attribute,
    span_map,
    tracker_context,
)
from burr.tracking.base import SyncTrackingClient
from burr.visibility import ActionSpan


class SampleModel(pydantic.BaseModel):
    foo: int
    bar: bool


@pytest.fixture(autouse=True)
def clear_span_map():
    """span_map is module-global, so keep a failing test from leaking entries into the next one."""
    span_map.clear()
    yield
    span_map.clear()


@pytest.mark.parametrize(
    "value, expected",
    [
        ("hello", "hello"),
        (1, 1),
        ((1, 1), [1, 1]),
        ((1.0, 1.0), [1.0, 1.0]),
        ((True, True), [True, True]),
        (("hello", "hello"), ["hello", "hello"]),
        (SampleModel(foo=1, bar=True), json.dumps(serde.serialize(SampleModel(foo=1, bar=True)))),
    ],
)
def test_convert_to_otel_attribute(value, expected):
    assert convert_to_otel_attribute(value) == expected


def test_burr_tracking_span_processor_on_start_with_none_tracker():
    """Test that on_start handles None tracker gracefully without raising an error."""
    processor = BurrTrackingSpanProcessor()

    # Mock a span with a parent
    mock_span = Mock(spec=Span)
    mock_span.parent = Mock()
    mock_span.parent.span_id = 12345
    mock_span.name = "test_span"

    # Mock the get_cached_span to return a parent span context
    with patch("burr.integrations.opentelemetry.get_cached_span") as mock_get_cached:
        mock_parent_context = Mock(spec=FullSpanContext)
        mock_parent_context.action_span = Mock(spec=ActionSpan)
        mock_spawned_span = Mock(spec=ActionSpan)
        mock_parent_context.action_span.spawn = Mock(return_value=mock_spawned_span)
        mock_parent_context.partition_key = "test_partition"
        mock_parent_context.app_id = "test_app"
        mock_parent_context.tracker = None
        mock_get_cached.return_value = mock_parent_context

        # Mock cache_span
        with patch("burr.integrations.opentelemetry.cache_span"):
            # Set tracker_context to None (simulating no tracker in context)
            token = tracker_context.set(None)
            try:
                # This should not raise an error even though tracker is None
                processor.on_start(mock_span, parent_context=None)
            finally:
                tracker_context.reset(token)


def test_burr_tracking_span_processor_on_end_with_none_tracker():
    """Test that on_end handles None tracker gracefully without raising an error."""
    processor = BurrTrackingSpanProcessor()

    # Mock a span
    mock_span = Mock(spec=Span)
    mock_span_context = Mock(spec=SpanContext)
    mock_span_context.span_id = 67890
    mock_span.get_span_context = Mock(return_value=mock_span_context)
    mock_span.attributes = {}

    # Mock the get_cached_span to return a cached span
    with patch("burr.integrations.opentelemetry.get_cached_span") as mock_get_cached:
        mock_cached_span = Mock(spec=FullSpanContext)
        mock_cached_span.action_span = Mock(spec=ActionSpan)
        mock_cached_span.action_span.action = "test_action"
        mock_cached_span.action_span.action_sequence_id = 1
        mock_cached_span.app_id = "test_app"
        mock_cached_span.partition_key = "test_partition"
        mock_cached_span.tracker = None
        mock_get_cached.return_value = mock_cached_span

        # Mock uncache_span
        with patch("burr.integrations.opentelemetry.uncache_span") as mock_uncache:
            # Set tracker_context to None (simulating no tracker in context)
            token = tracker_context.set(None)
            try:
                # This should not raise an error even though tracker is None
                processor.on_end(mock_span)
            finally:
                tracker_context.reset(token)
            # The span is still removed from the cache when no tracker is found
            mock_uncache.assert_called_once_with(mock_span)


def test_burr_tracking_span_processor_on_start_with_valid_tracker():
    """Test that on_start calls tracker methods when tracker is available."""
    processor = BurrTrackingSpanProcessor()

    # Mock a span with a parent
    mock_span = Mock(spec=Span)
    mock_span.parent = Mock()
    mock_span.parent.span_id = 12345
    mock_span.name = "test_span"

    # Mock tracker
    mock_tracker = Mock(spec=SyncTrackingClient)

    # Mock the get_cached_span to return a parent span context
    with patch("burr.integrations.opentelemetry.get_cached_span") as mock_get_cached:
        mock_parent_context = Mock(spec=FullSpanContext)
        mock_parent_action_span = Mock(spec=ActionSpan)
        mock_spawned_span = Mock(spec=ActionSpan)
        mock_spawned_span.action = "test_action"
        mock_spawned_span.action_sequence_id = 1
        mock_parent_action_span.spawn = Mock(return_value=mock_spawned_span)
        mock_parent_context.action_span = mock_parent_action_span
        mock_parent_context.partition_key = "test_partition"
        mock_parent_context.app_id = "test_app"
        mock_get_cached.return_value = mock_parent_context

        # Mock cache_span
        with patch("burr.integrations.opentelemetry.cache_span"):
            # Set tracker_context to a valid tracker
            token = tracker_context.set(mock_tracker)
            try:
                processor.on_start(mock_span, parent_context=None)

                # Verify that pre_start_span was called on the tracker
                assert mock_tracker.pre_start_span.called
            finally:
                tracker_context.reset(token)


def test_burr_tracking_span_processor_on_end_with_valid_tracker():
    """Test that on_end calls tracker methods when tracker is available."""
    processor = BurrTrackingSpanProcessor()

    # Mock a span
    mock_span = Mock(spec=Span)
    mock_span_context = Mock(spec=SpanContext)
    mock_span_context.span_id = 67890
    mock_span.get_span_context = Mock(return_value=mock_span_context)
    mock_span.attributes = {}

    # Mock tracker
    mock_tracker = Mock(spec=SyncTrackingClient)

    # Mock the get_cached_span to return a cached span
    with patch("burr.integrations.opentelemetry.get_cached_span") as mock_get_cached:
        mock_cached_span = Mock(spec=FullSpanContext)
        mock_cached_span.action_span = Mock(spec=ActionSpan)
        mock_cached_span.action_span.action = "test_action"
        mock_cached_span.action_span.action_sequence_id = 1
        mock_cached_span.app_id = "test_app"
        mock_cached_span.partition_key = "test_partition"
        mock_get_cached.return_value = mock_cached_span

        # Mock uncache_span
        with patch("burr.integrations.opentelemetry.uncache_span"):
            # Set tracker_context to a valid tracker
            token = tracker_context.set(mock_tracker)
            try:
                processor.on_end(mock_span)

                # Verify that post_end_span was called on the tracker
                assert mock_tracker.post_end_span.called
            finally:
                tracker_context.reset(token)


def _tracker_with_local_provider():
    provider = TracerProvider()
    provider.add_span_processor(BurrTrackingSpanProcessor())
    tracer = provider.get_tracer("test")
    burr_tracker = Mock(spec=SyncTrackingClient)
    with patch("burr.integrations.opentelemetry.initialize_tracer"):
        otel_tracker = OpenTelemetryTracker(burr_tracker=burr_tracker)
    otel_tracker.tracer = tracer
    action = Mock(spec=Action)
    action.name = "work"
    return tracer, burr_tracker, otel_tracker, action


def _pre_run_step(otel_tracker, action):
    otel_tracker.pre_run_step(
        app_id="test_app",
        partition_key="test_partition",
        sequence_id=0,
        state=State({}),
        action=action,
        inputs={},
    )


def _post_run_step(otel_tracker, action):
    otel_tracker.post_run_step(
        app_id="test_app",
        partition_key="test_partition",
        sequence_id=0,
        state=State({}),
        action=action,
        result={},
        exception=None,
    )


@pytest.mark.parametrize("on_worker_thread", [False, True])
def test_burr_tracking_span_processor_child_span_reaches_tracker_and_is_uncached(on_worker_thread):
    """Spans started inside an action are logged to the Burr tracker and removed from the span
    cache, whether they end on the action's thread or on a worker thread that only carries the
    OpenTelemetry context (thread pools don't copy context vars, so tracker_context is unset there).
    """
    tracer, burr_tracker, otel_tracker, action = _tracker_with_local_provider()
    child_span_ids = []

    def child():
        with tracer.start_as_current_span("llm_call") as span:
            span.set_attribute("model", "test-model")
            child_span_ids.append(span.get_span_context().span_id)
            assert child_span_ids[-1] in span_map

    def run_step():
        _pre_run_step(otel_tracker, action)
        if on_worker_thread:
            ctx = otel_context.get_current()

            def run_with_otel_context():
                token = otel_context.attach(ctx)
                try:
                    child()
                finally:
                    otel_context.detach(token)

            with ThreadPoolExecutor(max_workers=1) as executor:
                executor.submit(run_with_otel_context).result()
        else:
            child()
        _post_run_step(otel_tracker, action)

    contextvars.copy_context().run(run_step)

    assert len(child_span_ids) == 1
    assert child_span_ids[0] not in span_map
    assert len(span_map) == 0
    burr_tracker.pre_start_span.assert_called_once()
    assert burr_tracker.pre_start_span.call_args.kwargs["span"].name == "llm_call"
    # one for the child span, one for the action span itself
    assert burr_tracker.post_end_span.call_count == 2
    burr_tracker.do_log_attributes.assert_called_once()
    assert burr_tracker.do_log_attributes.call_args.kwargs["attributes"] == {"model": "test-model"}


def test_burr_tracking_span_processor_child_span_ending_after_step_is_uncached():
    """A span started during an action but ended after post_run_step (which clears
    tracker_context) is still logged to the owning tracker and removed from the span cache."""
    tracer, burr_tracker, otel_tracker, action = _tracker_with_local_provider()

    def run_step():
        _pre_run_step(otel_tracker, action)
        span = tracer.start_span("background_call")
        _post_run_step(otel_tracker, action)
        assert tracker_context.get() is None
        span.end()
        return span.get_span_context().span_id

    span_id = contextvars.copy_context().run(run_step)

    assert span_id not in span_map
    assert len(span_map) == 0
    assert burr_tracker.post_end_span.call_args.kwargs["span"].name == "background_call"


def test_burr_tracking_span_processor_uncaches_span_when_tracker_raises():
    """A failing tracker callback in on_end must not leave the span in the cache."""
    tracer, burr_tracker, otel_tracker, action = _tracker_with_local_provider()
    burr_tracker.post_end_span.side_effect = RuntimeError("tracker failed")
    span_ids = []

    def run_step():
        _pre_run_step(otel_tracker, action)
        span = tracer.start_span("llm_call")
        span_ids.append(span.get_span_context().span_id)
        with pytest.raises(RuntimeError, match="tracker failed"):
            span.end()
        burr_tracker.post_end_span.side_effect = None
        _post_run_step(otel_tracker, action)

    contextvars.copy_context().run(run_step)

    assert span_ids[0] not in span_map
    assert len(span_map) == 0


def test_burr_tracking_span_processor_span_after_nested_app_step_reaches_outer_tracker():
    """A sub-application stepped inside an action clears tracker_context in its post_run_step.
    Spans the outer action opens afterwards still reach the outer tracker and leave the cache."""
    tracer, outer_tracker, outer_otel, outer_action = _tracker_with_local_provider()
    _, inner_tracker, inner_otel, inner_action = _tracker_with_local_provider()
    inner_otel.tracer = tracer

    def run_step():
        _pre_run_step(outer_otel, outer_action)
        _pre_run_step(inner_otel, inner_action)
        _post_run_step(inner_otel, inner_action)
        assert tracker_context.get() is None
        with tracer.start_as_current_span("after_inner"):
            pass
        _post_run_step(outer_otel, outer_action)

    contextvars.copy_context().run(run_step)

    assert len(span_map) == 0
    ended = [c.kwargs["span"].name for c in outer_tracker.post_end_span.call_args_list]
    assert "after_inner" in ended
    assert "after_inner" not in [
        c.kwargs["span"].name for c in inner_tracker.post_end_span.call_args_list
    ]