Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

Fix: openlit open webui pipelines (#1052)

noy-solvin4b56a2349cee
diff --git a/examples/test_qdrant_compatibility.py b/examples/test_qdrant_compatibility.py
index 646e86da..1e8f4cb2 100644
--- a/examples/test_qdrant_compatibility.py
+++ b/examples/test_qdrant_compatibility.py
@@ -1,6 +1,7 @@
"""
Test script to reproduce Issue #971 - Qdrant instrumentation failure with qdrant-client >= 1.16.0
"""
+
import sys
print("=" * 60)
@@ -10,6 +11,7 @@ print("=" * 60)
# Check qdrant-client version
try:
import importlib.metadata
+
qdrant_version = importlib.metadata.version("qdrant-client")
print(f"✓ qdrant-client version: {qdrant_version}")
except Exception as e:
@@ -19,21 +21,22 @@ except Exception as e:
# Check if deprecated methods exist
try:
from qdrant_client import QdrantClient
+
deprecated = ["search", "search_groups", "recommend"]
new_methods = ["query_points", "query_points_groups", "query_batch_points"]
-
+
print(f"\nDeprecated methods availability:")
for method in deprecated:
has_method = hasattr(QdrantClient, method)
status = "✓" if has_method else "✗"
print(f" {status} {method}: {has_method}")
-
+
print(f"\nNew methods availability:")
for method in new_methods:
has_method = hasattr(QdrantClient, method)
status = "✓" if has_method else "✗"
print(f" {status} {method}: {has_method}")
-
+
except Exception as e:
print(f"✗ Error checking QdrantClient methods: {e}")
sys.exit(1)
@@ -45,16 +48,18 @@ print("=" * 60)
try:
import openlit
+
openlit.init(
otlp_endpoint="http://localhost:4318",
collect_gpu_stats=False,
- disable_batch=True
+ disable_batch=True,
)
print("✓ OpenLIT instrumentation initialized successfully!")
-
+
except Exception as e:
print(f"✗ OpenLIT instrumentation FAILED: {e}")
import traceback
+
traceback.print_exc()
sys.exit(1)
diff --git a/sdk/python/src/openlit/instrumentation/litellm/__init__.py b/sdk/python/src/openlit/instrumentation/litellm/__init__.py
index 7ded53cf..3e0ee0b5 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/__init__.py
@@ -30,73 +30,74 @@ class LiteLLMInstrumentor(BaseInstrumentor):
event_provider = kwargs.get("event_provider")
version = importlib.metadata.version("litellm")
- # Chat completions
- wrap_function_wrapper(
- "litellm",
- "completion",
- completion(
- version,
- environment,
- application_name,
- tracer,
- pricing_info,
- capture_message_content,
- metrics,
- disable_metrics,
- event_provider,
- ),
- )
+ for target_module in ["litellm", "litellm.main"]:
+ # Chat completions
+ wrap_function_wrapper(
+ target_module,
+ "completion",
+ completion(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ event_provider,
+ ),
+ )
- # Async chat completions
- wrap_function_wrapper(
- "litellm",
- "acompletion",
- acompletion(
- version,
- environment,
- application_name,
- tracer,
- pricing_info,
- capture_message_content,
- metrics,
- disable_metrics,
- event_provider,
- ),
- )
+ # Async chat completions
+ wrap_function_wrapper(
+ target_module,
+ "acompletion",
+ acompletion(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ event_provider,
+ ),
+ )
- # Embeddings
- wrap_function_wrapper(
- "litellm",
- "embedding",
- embedding(
- version,
- environment,
- application_name,
- tracer,
- pricing_info,
- capture_message_content,
- metrics,
- disable_metrics,
- event_provider,
- ),
- )
+ # Embeddings
+ wrap_function_wrapper(
+ target_module,
+ "embedding",
+ embedding(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ event_provider,
+ ),
+ )
- # Async embeddings
- wrap_function_wrapper(
- "litellm",
- "aembedding",
- aembedding(
- version,
- environment,
- application_name,
- tracer,
- pricing_info,
- capture_message_content,
- metrics,
- disable_metrics,
- event_provider,
- ),
- )
+ # Async embeddings
+ wrap_function_wrapper(
+ target_module,
+ "aembedding",
+ aembedding(
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ event_provider,
+ ),
+ )
def _uninstrument(self, **kwargs):
pass
diff --git a/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py b/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
index 3e2e6237..298235f3 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
@@ -3,6 +3,7 @@ Module for monitoring LiteLLM API calls (async version).
"""
import time
+import asyncio
from opentelemetry.trace import SpanKind
from openlit.__helpers import (
handle_exception,
@@ -115,6 +116,28 @@ def acompletion(
self._span.end()
raise
+ except (GeneratorExit, asyncio.CancelledError):
+ try:
+ # Use the existing span that was started when the stream began
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=self._event_provider,
+ )
+ # End the span after processing
+ self._span.end()
+
+ except Exception as e:
+ handle_exception(self._span, e)
+ self._span.end()
+
+ raise
async def wrapper(wrapped, instance, args, kwargs):
"""
diff --git a/sdk/python/tests/test_agent_metrics.py b/sdk/python/tests/test_agent_metrics.py
index b684e16c..39158dd0 100644
--- a/sdk/python/tests/test_agent_metrics.py
+++ b/sdk/python/tests/test_agent_metrics.py
@@ -41,6 +41,7 @@ openlit.init(
# 1. ContextVar propagation
# ---------------------------------------------------------------------------
+
class TestAgentContext:
"""Tests for agent_context() and the underlying ContextVar."""
@@ -90,6 +91,7 @@ class TestAgentContext:
# 2. create_metrics_attributes — include_agent_name gating
# ---------------------------------------------------------------------------
+
class TestCreateMetricsAttributes:
"""Tests for agent name inclusion/exclusion on metric attributes."""
@@ -108,7 +110,8 @@ class TestCreateMetricsAttributes:
"""When include_agent_name=True and agent is set, attribute is present."""
with openlit.agent_context("my_agent"):
attrs = create_metrics_attributes(
- **self.BASE_KWARGS, include_agent_name=True,
+ **self.BASE_KWARGS,
+ include_agent_name=True,
)
assert attrs[SemanticConvention.GEN_AI_AGENT_NAME] == "my_agent"
@@ -124,7 +127,8 @@ class TestCreateMetricsAttributes:
"""When include_agent_name=False, attribute is absent even with context."""
with openlit.agent_context("my_agent"):
attrs = create_metrics_attributes(
- **self.BASE_KWARGS, include_agent_name=False,
+ **self.BASE_KWARGS,
+ include_agent_name=False,
)
assert SemanticConvention.GEN_AI_AGENT_NAME not in attrs
@@ -133,7 +137,8 @@ class TestCreateMetricsAttributes:
"""When include_agent_name=True but no context set, attribute is absent."""
assert get_agent_name() is None
attrs = create_metrics_attributes(
- **self.BASE_KWARGS, include_agent_name=True,
+ **self.BASE_KWARGS,
+ include_agent_name=True,
)
assert SemanticConvention.GEN_AI_AGENT_NAME not in attrs
@@ -224,6 +229,7 @@ class TestCreateMetricsAttributes:
# 3. record_agent_duration
# ---------------------------------------------------------------------------
+
class TestRecordAgentDuration:
"""Tests for record_agent_duration helper."""
@@ -245,8 +251,11 @@ class TestRecordAgentDuration:
fake_histogram = MagicMock()
metrics = {"genai_agent_operation_duration": fake_histogram}
record_agent_duration(
- metrics, "test_agent", 2.0,
- operation="invoke_agent", system="anthropic",
+ metrics,
+ "test_agent",
+ 2.0,
+ operation="invoke_agent",
+ system="anthropic",
)
_, attrs = fake_histogram.record.call_args[0]
@@ -257,7 +266,10 @@ class TestRecordAgentDuration:
fake_histogram = MagicMock()
metrics = {"genai_agent_operation_duration": fake_histogram}
record_agent_duration(
- metrics, "test_agent", 0.5, error_type="ValueError",
+ metrics,
+ "test_agent",
+ 0.5,
+ error_type="ValueError",
)
_, attrs = fake_histogram.record.call_args[0]
@@ -276,6 +288,7 @@ class TestRecordAgentDuration:
# 4. record_agent_invocation
# ---------------------------------------------------------------------------
+
class TestRecordAgentInvocation:
"""Tests for record_agent_invocation helper."""
@@ -297,7 +310,10 @@ class TestRecordAgentInvocation:
fake_counter = MagicMock()
metrics = {"genai_agent_invocations": fake_counter}
record_agent_invocation(
- metrics, "dispatcher", "another_agent", system="anthropic",
+ metrics,
+ "dispatcher",
+ "another_agent",
+ system="anthropic",
)
_, attrs = fake_counter.add.call_args[0]
@@ -316,6 +332,7 @@ class TestRecordAgentInvocation:
# 5. log_agent_invocation public API
# ---------------------------------------------------------------------------
+
class TestLogAgentInvocation:
"""Tests for the public openlit.log_agent_invocation() wrapper."""
@@ -362,6 +379,7 @@ class TestLogAgentInvocation:
# 6. Automatic agent-to-agent invocation detection
# ---------------------------------------------------------------------------
+
class TestNestedAgentDetection:
"""Tests that verify automatic invocation detection when agents nest."""
@@ -491,13 +509,16 @@ class TestNestedAgentDetection:
asyncio.run(run_test())
- assert not interference_detected, "Concurrent agents interfered with each other's context"
+ assert not interference_detected, (
+ "Concurrent agents interfered with each other's context"
+ )
# ---------------------------------------------------------------------------
# 7. record_agent_tool_error
# ---------------------------------------------------------------------------
+
class TestRecordAgentToolError:
"""Tests for record_agent_tool_error helper."""
@@ -520,8 +541,11 @@ class TestRecordAgentToolError:
fake_counter = MagicMock()
metrics = {"genai_agent_tool_errors": fake_counter}
record_agent_tool_error(
- metrics, "cart_agent", "add_to_cart",
- system="anthropic", model="claude-haiku-4-5",
+ metrics,
+ "cart_agent",
+ "add_to_cart",
+ system="anthropic",
+ model="claude-haiku-4-5",
)
_, attrs = fake_counter.add.call_args[0]
@@ -533,7 +557,10 @@ class TestRecordAgentToolError:
fake_counter = MagicMock()
metrics = {"genai_agent_tool_errors": fake_counter}
record_agent_tool_error(
- metrics, "product_agent", "search_products", system="openai",
+ metrics,
+ "product_agent",
+ "search_products",
+ system="openai",
)
_, attrs = fake_counter.add.call_args[0]
@@ -553,6 +580,7 @@ class TestRecordAgentToolError:
# 8. log_agent_tool_error public API
# ---------------------------------------------------------------------------
+
class TestLogAgentToolError:
"""Tests for the public openlit.log_agent_tool_error() wrapper."""
@@ -579,8 +607,10 @@ class TestLogAgentToolError:
openlit.OpenlitConfig.metrics_dict = {"genai_agent_tool_errors": fake_counter}
try:
openlit.log_agent_tool_error(
- "cart_agent", "get_cart",
- system="anthropic", model="claude-haiku-4-5",
+ "cart_agent",
+ "get_cart",
+ system="anthropic",
+ model="claude-haiku-4-5",
)
_, attrs = fake_counter.add.call_args[0]
@@ -603,6 +633,7 @@ class TestLogAgentToolError:
# 7. Agent error tracking on duration metric
# ---------------------------------------------------------------------------
+
class TestAgentErrorTracking:
"""Tests that agent errors are recorded on the duration histogram via error_type."""
@@ -611,8 +642,11 @@ class TestAgentErrorTracking:
fake_histogram = MagicMock()
metrics = {"genai_agent_operation_duration": fake_histogram}
record_agent_duration(
- metrics, "failing_agent", 0.8,
- operation="chat", error_type="ValueError",
+ metrics,
+ "failing_agent",
+ 0.8,
+ operation="chat",
+ error_type="ValueError",
)
fake_histogram.record.assert_called_once()
@@ -635,7 +669,10 @@ class TestAgentErrorTracking:
fake_histogram = MagicMock()
metrics = {"genai_agent_operation_duration": fake_histogram}
record_agent_duration(
- metrics, "agent", 1.0, error_type=None,
+ metrics,
+ "agent",
+ 1.0,
+ error_type=None,
)
_, attrs = fake_histogram.record.call_args[0]
diff --git a/sdk/python/tests/test_litellm_reproduction.py b/sdk/python/tests/test_litellm_reproduction.py
new file mode 100644
index 00000000..47a13c5f
--- /dev/null
+++ b/sdk/python/tests/test_litellm_reproduction.py
@@ -0,0 +1,88 @@
+"""Tests for LiteLLM instrumentation reproduction."""
+
+import asyncio
+from unittest.mock import MagicMock
+
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from wrapt import FunctionWrapper
+import pytest
+
+import openlit
+from openlit.instrumentation.litellm.async_litellm import (
+ acompletion as openlit_acompletion,
+)
+
+
+async def test_litellm_async_stream_premature_close_reproduction():
+ """
+ Reproduces the issue where premature async stream cancellation (e.g. CancelledError)
+ skips the metric recording and span closure in OpenLIT's LiteLLM instrumentation.
+ """
+ # Setup OTEL in-memory exporter
+ exporter = InMemorySpanExporter()
+ tracer_provider = TracerProvider()
+ tracer_processor = SimpleSpanProcessor(exporter)
+ tracer_provider.add_span_processor(tracer_processor)
+
+ # We test the wrapper factory directly
+ wrapper_factory = openlit_acompletion(
+ version="1.0.0",
+ environment="test",
+ application_name="test",
+ tracer=tracer_provider.get_tracer("test"),
+ pricing_info={},
+ capture_message_content=True,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ # Original function returns an async generator that raises CancelledError
+ async def mock_original_gen():
+ yield MagicMock()
+ raise asyncio.CancelledError()
+
+ async def mock_original_acompletion(*args, **kwargs):
+ return mock_original_gen()
+
+ # Apply the wrapper
+ wrapped_stream = await wrapper_factory(
+ mock_original_acompletion, None, [], {"model": "gpt-4", "stream": True}
+ )
+
+ # Consume the first chunk
+ await anext(wrapped_stream)
+
+ # The next call will raise CancelledError
+ with pytest.raises(asyncio.CancelledError):
+ await anext(wrapped_stream)
+
+ # Check finished spans
+ spans = exporter.get_finished_spans()
+
+ # This should fail currently because spans will be 0 due to the bug
+ # The span should have been closed even on cancellation/error
+ assert len(spans) == 1, (
+ f"Reproduction successful: Expected 1 finished span, but got {len(spans)}"
+ )
+
+
+async def test_litellm_main_patch_missing_reproduction():
+ """
+ Reproduces the issue where litellm.main.acompletion is not instrumented.
+ """
+ import litellm.main
+
+ # Initialize OpenLIT
+ openlit.init()
+
+ # Check if litellm.main.acompletion is instrumented via wrapt.FunctionWrapper
+ is_instrumented = isinstance(litellm.main.acompletion, FunctionWrapper)
+
+ # This should fail currently because it's not instrumented due to the bug
+ assert is_instrumented, (
+ "Reproduction successful: litellm.main.acompletion is NOT instrumented by OpenLIT!"
+ )