Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
15.5 KB
Raw Blame
"""Tests for backend LLM tracing (`observability/tracing.py`).

Tracing is off by default, and when it is on, traces carry metadata only unless
content capture is explicitly opted into (twice, in SaaS). Both failure modes
are silent — tracing that never starts, or prompts and chunk text leaking into
Langfuse — so the gate and the redaction switch are pinned here.
"""

import pytest

from observability import spans, tracing
from observability.tracing import TRACING_CALLBACK, configure_tracing, content_capture_enabled

_TRACING_ENV = (
    "OPENRAG_TRACING_ENABLED",
    "OPENRAG_TRACING_CAPTURE_CONTENT",
    "OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS",
    "OPENRAG_RUN_MODE",
    "LANGFUSE_PUBLIC_KEY",
    "LANGFUSE_SECRET_KEY",
    "LANGFUSE_HOST",
    "LANGFUSE_BASE_URL",
    "LANGFUSE_OTEL_HOST",
    "USE_OTEL_LITELLM_REQUEST_SPAN",
    "OTEL_TRACES_SAMPLER",
)


@pytest.fixture(autouse=True)
def _isolated_tracing(monkeypatch):
    """Clear tracing env and restore LiteLLM's process-global callback state."""
    import litellm

    for name in _TRACING_ENV:
        # setenv first so teardown restores the original value (or absence)
        # even when configure_tracing() writes the variable itself.
        monkeypatch.setenv(name, "")
        monkeypatch.delenv(name)
    # LiteLLM copies `callbacks` into these lists on first use and dedupes by
    # class, so a logger from one test would otherwise swallow the next test's.
    for name in (
        "callbacks",
        "input_callback",
        "success_callback",
        "failure_callback",
        "_async_success_callback",
        "_async_failure_callback",
    ):
        monkeypatch.setattr(litellm, name, [])
    monkeypatch.setattr(litellm, "turn_off_message_logging", False)
    monkeypatch.setattr(tracing, "_enabled", False)
    # configure_tracing() builds a real Langfuse client for our spans; never
    # leave one behind for later tests to export through.
    monkeypatch.setattr(spans, "_client", None)


def _langfuse_loggers() -> list:
    """The Langfuse OTel loggers registered on LiteLLM."""
    import litellm
    from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger

    return [cb for cb in litellm.callbacks if isinstance(cb, LangfuseOtelLogger)]


@pytest.fixture
def keys(monkeypatch):
    monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-test")
    monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-test")


def test_disabled_by_default_registers_no_callback(keys):
    import litellm

    assert configure_tracing() is False
    assert TRACING_CALLBACK not in litellm.callbacks
    assert _langfuse_loggers() == []


@pytest.mark.parametrize("value", ["false", "0", "", "no"])
def test_falsy_flag_registers_no_callback(monkeypatch, keys, value):
    import litellm

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", value)

    assert configure_tracing() is False
    assert TRACING_CALLBACK not in litellm.callbacks
    assert _langfuse_loggers() == []


def test_enabled_without_keys_warns_and_stays_off(monkeypatch, caplog):
    import litellm

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
    warnings = []
    monkeypatch.setattr(tracing.logger, "warning", lambda msg, **kw: warnings.append(msg))

    assert configure_tracing() is False
    assert TRACING_CALLBACK not in litellm.callbacks
    assert _langfuse_loggers() == []
    assert any("LANGFUSE_PUBLIC_KEY" in w for w in warnings)


def test_enabled_with_keys_registers_langfuse_otel_callback(monkeypatch, keys):

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")

    assert configure_tracing() is True
    (logger,) = _langfuse_loggers()
    # Registered as LiteLLM's langfuse_otel, which keeps it off the global provider.
    assert logger.callback_name == TRACING_CALLBACK
    assert logger._skip_set_global() is True


def test_registration_is_idempotent(monkeypatch, keys):
    """Startup can run more than once in a process (tests, reloads)."""

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")

    configure_tracing()
    configure_tracing()

    assert len(_langfuse_loggers()) == 1


def test_enabled_redacts_message_content_by_default(monkeypatch, keys):
    import litellm

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")

    configure_tracing()

    assert litellm.turn_off_message_logging is True


def test_capture_opt_in_keeps_content_in_oss(monkeypatch, keys):
    import litellm

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
    monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", "true")

    configure_tracing()

    assert litellm.turn_off_message_logging is False


@pytest.mark.parametrize(
    ("run_mode", "capture", "allow_saas", "expected"),
    [
        ("oss", "false", "false", False),
        ("oss", "true", "false", True),
        ("on_prem", "true", "false", True),
        ("saas", "true", "false", False),
        ("saas", "false", "true", False),
        ("saas", "true", "true", True),
    ],
)
def test_content_capture_policy(monkeypatch, run_mode, capture, allow_saas, expected):
    monkeypatch.setenv("OPENRAG_RUN_MODE", run_mode)
    monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", capture)
    monkeypatch.setenv("OPENRAG_TRACING_ALLOW_CONTENT_IN_SAAS", allow_saas)

    assert content_capture_enabled() is expected


def test_saas_without_second_flag_redacts(monkeypatch, keys):
    import litellm

    monkeypatch.setenv("OPENRAG_RUN_MODE", "saas")
    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
    monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", "true")

    configure_tracing()

    assert litellm.turn_off_message_logging is True


def test_empty_host_targets_the_same_langfuse_cloud_as_langflow(monkeypatch, keys):
    """LiteLLM defaults to the US cloud; Langflow and the Langfuse SDK default to EU."""
    from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")

    configure_tracing()

    config = LangfuseOtelLogger.get_langfuse_otel_config()
    assert config.endpoint == "https://cloud.langfuse.com/api/public/otel"


def test_explicit_host_is_respected(monkeypatch, keys):
    from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
    monkeypatch.setenv("LANGFUSE_HOST", "http://langfuse.internal:3000")

    configure_tracing()

    config = LangfuseOtelLogger.get_langfuse_otel_config()
    assert config.endpoint == "http://langfuse.internal:3000/api/public/otel"


def test_base_url_is_used_as_the_host(monkeypatch, keys):
    """Langfuse's setup snippets name the host LANGFUSE_BASE_URL; LiteLLM reads LANGFUSE_HOST."""
    from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
    monkeypatch.setenv("LANGFUSE_BASE_URL", "https://us.cloud.langfuse.com")

    configure_tracing()

    config = LangfuseOtelLogger.get_langfuse_otel_config()
    assert config.endpoint == "https://us.cloud.langfuse.com/api/public/otel"


def test_base_url_wins_over_host_like_langflow(monkeypatch, keys):
    from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
    monkeypatch.setenv("LANGFUSE_BASE_URL", "https://us.cloud.langfuse.com")
    monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com")

    configure_tracing()

    config = LangfuseOtelLogger.get_langfuse_otel_config()
    assert config.endpoint == "https://us.cloud.langfuse.com/api/public/otel"


def test_missing_tracing_extra_warns_and_stays_off(monkeypatch, keys):
    import builtins

    import litellm

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")
    real_import = builtins.__import__

    def _no_otel_sdk(name, *args, **kwargs):
        if name.startswith("opentelemetry.sdk") or name.startswith("opentelemetry.exporter"):
            raise ImportError(name)
        return real_import(name, *args, **kwargs)

    monkeypatch.setattr(builtins, "__import__", _no_otel_sdk)
    warnings = []
    monkeypatch.setattr(tracing.logger, "warning", lambda msg, **kw: warnings.append(msg))

    assert configure_tracing() is False
    assert TRACING_CALLBACK not in litellm.callbacks
    assert _langfuse_loggers() == []
    assert any("--extra tracing" in w for w in warnings)


# ── LiteLLM contract ───────────────────────────────────────────────────────
# Run real LiteLLM calls (mock_response, no network) through the Langfuse OTel
# logger with an in-memory exporter in place of OTLP. These pin the pinned
# LiteLLM's behaviour that tracing relies on: redaction, nesting under the
# caller's active span, and failures recorded on the span.

_SECRETS = ("SECRET-PROMPT", "SECRET-ANSWER", "SECRET-CHUNK")


@pytest.fixture
def traced(monkeypatch, keys):
    """Configure tracing, then swap the OTLP-exporting callback for an in-memory one."""
    import litellm
    from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
    from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter

    from observability import litellm_langfuse

    monkeypatch.setenv("OPENRAG_TRACING_ENABLED", "true")

    def _configure():
        exporter = InMemorySpanExporter()
        # configure_tracing() builds its LiteLLM logger through this; hand it
        # an in-memory exporter instead of OTLP to Langfuse.
        real_build = litellm_langfuse.build_langfuse_logger
        monkeypatch.setattr(
            litellm_langfuse,
            "build_langfuse_logger",
            lambda **kwargs: real_build(**kwargs, exporter=exporter),
        )
        assert configure_tracing() is True
        assert any(isinstance(cb, LangfuseOtelLogger) for cb in litellm.callbacks)
        return exporter

    return _configure


@pytest.fixture
def caller_tracer():
    from opentelemetry.sdk.trace import TracerProvider

    return TracerProvider().get_tracer("openrag-test")


async def _drain(exporter, expected: int):
    """LiteLLM runs logging callbacks in the background; wait for the spans."""
    import asyncio

    for _ in range(100):
        spans = [s for s in exporter.get_finished_spans() if s.name == "litellm_request"]
        if len(spans) >= expected:
            return spans
        await asyncio.sleep(0.02)
    raise AssertionError(f"expected {expected} litellm_request spans, got {len(spans)}")


def _span_text(span) -> str:
    return (
        repr(dict(span.attributes))
        + repr([dict(e.attributes or {}) for e in span.events])
        + repr(span.status.description)
    )


async def _completion(content="SECRET-PROMPT", response="SECRET-ANSWER"):
    import litellm

    return await litellm.acompletion(
        model="openai/gpt-4o-mini",
        messages=[{"role": "user", "content": content}],
        mock_response=response,
    )


async def _embedding():
    import litellm

    return await litellm.aembedding(
        model="openai/text-embedding-3-small", input=["SECRET-CHUNK"], mock_response=[0.1, 0.2]
    )


@pytest.mark.asyncio
async def test_gateway_calls_are_traced_without_content_by_default(traced):
    exporter = traced()

    await _completion()
    await _embedding()

    spans = await _drain(exporter, 2)
    assert {span.attributes["llm.model_name"] for span in spans} == {
        "gpt-4o-mini",
        "text-embedding-3-small",
    }
    for span in spans:
        assert not any(secret in _span_text(span) for secret in _SECRETS)


@pytest.mark.asyncio
async def test_content_is_traced_when_capture_is_opted_in(monkeypatch, traced):
    monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", "true")
    exporter = traced()

    await _completion()

    (span,) = await _drain(exporter, 1)
    assert "SECRET-PROMPT" in _span_text(span)


@pytest.mark.asyncio
async def test_gateway_spans_nest_under_the_callers_active_span(traced, caller_tracer):
    """Ingest/search spans (03, 05) rely on this: no ids are threaded through call sites."""
    exporter = traced()

    with caller_tracer.start_as_current_span("ingest.embed") as parent:
        await _embedding()
        parent_ctx = parent.get_span_context()

    (span,) = await _drain(exporter, 1)
    assert span.context.trace_id == parent_ctx.trace_id
    assert span.parent.span_id == parent_ctx.span_id


@pytest.mark.asyncio
async def test_gateway_spans_survive_an_unsampled_foreign_parent(traced):
    """Instana's HTTP spans are active during requests and are never sampled.

    The SDK's default parent-based sampler would drop every LLM span beneath
    them, so Langfuse would see nothing whenever Instana is also enabled.
    """
    from opentelemetry import trace
    from opentelemetry.trace import NonRecordingSpan, SpanContext, TraceFlags

    exporter = traced()
    instana_like = NonRecordingSpan(
        SpanContext(
            trace_id=0xD20FD29B103E15D2,
            span_id=0xD20FD29B103E15D2,
            is_remote=False,
            trace_flags=TraceFlags(TraceFlags.DEFAULT),
        )
    )

    with trace.use_span(instana_like):
        await _embedding()

    (span,) = await _drain(exporter, 1)
    assert span.context.trace_id == 0xD20FD29B103E15D2


@pytest.mark.asyncio
async def test_failed_call_is_traced_with_the_error(traced):
    import litellm
    from opentelemetry.trace import StatusCode

    exporter = traced()

    with pytest.raises(litellm.InternalServerError):
        await _completion(response=Exception("upstream exploded"))

    (span,) = await _drain(exporter, 1)
    assert span.status.status_code is StatusCode.ERROR
    assert "error.type" in span.attributes
    # Provider error text can echo the prompt or chunk: content, so dropped
    # unless capture is on (turn_off_message_logging doesn't cover failures).
    assert "upstream exploded" not in _span_text(span)
    assert "error.message" not in span.attributes
    assert "error.stack_trace" not in span.attributes
    assert not any(event.name == "exception" for event in span.events)


@pytest.mark.asyncio
async def test_failed_call_keeps_the_error_text_when_capture_is_on(monkeypatch, traced):
    import litellm

    monkeypatch.setenv("OPENRAG_TRACING_CAPTURE_CONTENT", "true")
    exporter = traced()

    with pytest.raises(litellm.InternalServerError):
        await _completion(response=Exception("upstream exploded"))

    (span,) = await _drain(exporter, 1)
    assert "upstream exploded" in _span_text(span)


@pytest.mark.parametrize(
    ("env", "expected"),
    [
        ({}, "https://cloud.langfuse.com"),
        ({"LANGFUSE_HOST": "http://old:3000"}, "http://old:3000"),
        ({"LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com"}, "https://us.cloud.langfuse.com"),
        (
            {
                "LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com",
                "LANGFUSE_HOST": "http://old:3000",
            },
            "https://us.cloud.langfuse.com",
        ),
    ],
)
def test_langfuse_host_precedence(monkeypatch, env, expected):
    from config.settings import get_langfuse_host

    for name, value in env.items():
        monkeypatch.setenv(name, value)

    assert get_langfuse_host() == expected


def test_span_client_resolves_the_host_itself(monkeypatch, keys):
    """Our spans and LiteLLM's must export to the same Langfuse, whatever ran first."""
    import langfuse

    captured = {}

    class _FakeLangfuse:
        def __init__(self, **kwargs):
            captured.update(kwargs)

    monkeypatch.setattr(langfuse, "Langfuse", _FakeLangfuse)
    monkeypatch.setenv("LANGFUSE_BASE_URL", "https://us.cloud.langfuse.com")

    spans.init_span_client()

    assert captured["base_url"] == "https://us.cloud.langfuse.com"