Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(groq): end streaming span on early break/close (#1516)

Paco Cartones864a553b09a2
diff --git a/sdk/python/src/openlit/instrumentation/groq/async_groq.py b/sdk/python/src/openlit/instrumentation/groq/async_groq.py
index 686f4353..562f8199 100644
--- a/sdk/python/src/openlit/instrumentation/groq/async_groq.py
+++ b/sdk/python/src/openlit/instrumentation/groq/async_groq.py
@@ -9,6 +9,7 @@ from openlit.__helpers import (
handle_exception,
set_server_address_and_port,
record_completion_metrics,
+ safe_detach,
)
from openlit.instrumentation.groq.utils import (
process_chunk,
@@ -72,13 +73,25 @@ def async_chat(
self._tbt = 0
self._server_address = server_address
self._server_port = server_port
+ self._streaming_response_processed = False
async def __aenter__(self):
await self.__wrapped__.__aenter__()
return self
async def __aexit__(self, exc_type, exc_value, traceback):
- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+ try:
+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+ finally:
+ if exc_type:
+ self._streaming_response_processed = True
+ handle_exception(self._span, exc_value)
+ if self._span.is_recording():
+ self._span.end()
+ else:
+ # A break before exhaustion never hits StopAsyncIteration,
+ # so the span would leak without normal-exit finalization.
+ self._finalize_streaming_span()
def __aiter__(self):
return self
@@ -93,24 +106,42 @@ def async_chat(
process_chunk(self, chunk)
return chunk
except StopAsyncIteration:
- try:
- with self._span:
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=event_provider,
- )
+ self._finalize_streaming_span()
+ raise
- except Exception as e:
- handle_exception(self._span, e)
+ def _finalize_streaming_span(self):
+ """Complete and end the span exactly once.
- raise
+ Called on stream exhaustion and again from aclose()/manager exit;
+ the flag keeps the double call a no-op so early exits that never
+ see StopAsyncIteration still export the span instead of leaking it.
+ """
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ try:
+ with self._span:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=event_provider,
+ )
+
+ except Exception as e:
+ handle_exception(self._span, e)
+
+ async def aclose(self):
+ """Close the wrapped stream and finalize the span if not ended."""
+ try:
+ await self.__wrapped__.aclose()
+ finally:
+ self._finalize_streaming_span()
async def wrapper(wrapped, instance, args, kwargs):
"""
@@ -133,10 +164,10 @@ def async_chat(
awaited_wrapped = await wrapped(*args, **kwargs)
except Exception as e:
handle_exception(span, e)
- context_api.detach(token)
+ safe_detach(token)
span.end()
raise
- context_api.detach(token)
+ safe_detach(token)
return TracedAsyncStream(
awaited_wrapped, span, span_name, kwargs, server_address, server_port
)
diff --git a/sdk/python/src/openlit/instrumentation/groq/groq.py b/sdk/python/src/openlit/instrumentation/groq/groq.py
index e9492937..e89d413c 100644
--- a/sdk/python/src/openlit/instrumentation/groq/groq.py
+++ b/sdk/python/src/openlit/instrumentation/groq/groq.py
@@ -9,6 +9,7 @@ from openlit.__helpers import (
handle_exception,
set_server_address_and_port,
record_completion_metrics,
+ safe_detach,
)
from openlit.instrumentation.groq.utils import (
process_chunk,
@@ -72,13 +73,25 @@ def chat(
self._tbt = 0
self._server_address = server_address
self._server_port = server_port
+ self._streaming_response_processed = False
def __enter__(self):
self.__wrapped__.__enter__()
return self
def __exit__(self, exc_type, exc_value, traceback):
- self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+ try:
+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+ finally:
+ if exc_type:
+ self._streaming_response_processed = True
+ handle_exception(self._span, exc_value)
+ if self._span.is_recording():
+ self._span.end()
+ else:
+ # A break before exhaustion never hits StopIteration, so
+ # the span would leak without normal-exit finalization.
+ self._finalize_streaming_span()
def __iter__(self):
return self
@@ -93,24 +106,42 @@ def chat(
process_chunk(self, chunk)
return chunk
except StopIteration:
- try:
- with self._span:
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=event_provider,
- )
+ self._finalize_streaming_span()
+ raise
- except Exception as e:
- handle_exception(self._span, e)
+ def _finalize_streaming_span(self):
+ """Complete and end the span exactly once.
- raise
+ Called on stream exhaustion and again from close()/manager exit;
+ the flag keeps the double call a no-op so early exits that never
+ see StopIteration still export the span instead of leaking it.
+ """
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ try:
+ with self._span:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=event_provider,
+ )
+
+ except Exception as e:
+ handle_exception(self._span, e)
+
+ def close(self):
+ """Close the wrapped stream and finalize the span if not ended."""
+ try:
+ self.__wrapped__.close()
+ finally:
+ self._finalize_streaming_span()
def wrapper(wrapped, instance, args, kwargs):
"""
@@ -133,10 +164,10 @@ def chat(
awaited_wrapped = wrapped(*args, **kwargs)
except Exception as e:
handle_exception(span, e)
- context_api.detach(token)
+ safe_detach(token)
span.end()
raise
- context_api.detach(token)
+ safe_detach(token)
return TracedSyncStream(
awaited_wrapped, span, span_name, kwargs, server_address, server_port
)
diff --git a/sdk/python/tests/test_groq_stream_span_lifecycle.py b/sdk/python/tests/test_groq_stream_span_lifecycle.py
new file mode 100644
index 00000000..f0f8dc0c
--- /dev/null
+++ b/sdk/python/tests/test_groq_stream_span_lifecycle.py
@@ -0,0 +1,264 @@
+# pylint: disable=protected-access, duplicate-code, missing-function-docstring
+"""Regression tests: the Groq streaming wrappers must end their span on every
+exit path.
+
+`TracedSyncStream.__exit__`/`TracedAsyncStream.__aexit__` merely forwarded to
+the wrapped stream and there was no `close()`/`aclose()`, so the span was
+ended only inside the `except StopIteration`/`StopAsyncIteration` handler.
+A caller that `break`s out of `with … as stream:` before the stream is
+exhausted, or calls `stream.close()` early, never hit that handler, so the
+span stayed recording forever and was never exported — the whole call
+(span, cost, tokens) was lost. This is the Groq instance of the early-close
+streaming-span leak fixed for Anthropic in #1461 and filed for OpenAI in
+#1454/#1455.
+
+These tests drive the real `chat`/`async_chat` wrapper factories with a
+synthetic Groq-shaped stream and assert the span ends exactly once, carrying
+token-usage attributes, on each exit path.
+"""
+
+import time
+
+import pytest
+from opentelemetry import trace as trace_api, context as context_api
+from opentelemetry.trace import StatusCode
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.groq import groq as sync_mod
+from openlit.instrumentation.groq import async_groq as async_mod
+from openlit.semcov import SemanticConvention
+
+REQUEST_KWARGS = {
+ "model": "llama-3.1-8b-instant",
+ "stream": True,
+ "messages": [{"role": "user", "content": "Monitor LLM Applications"}],
+}
+
+# Two Groq-shaped chunks: a content delta, then a final chunk carrying usage.
+CHUNKS = [
+ {"choices": [{"delta": {"content": "partial"}, "finish_reason": None}]},
+ {
+ "choices": [{"delta": {"content": " answer"}, "finish_reason": "stop"}],
+ "x_groq": {
+ "id": "chatcmpl-1",
+ "model": "llama-3.1-8b-instant",
+ "system_fingerprint": "fp_test",
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5},
+ },
+ },
+]
+
+
+def _tracer_with_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return provider.get_tracer(__name__), exporter
+
+
+def _factory(tracer, *, is_async):
+ mod = async_mod if is_async else sync_mod
+ make = mod.async_chat if is_async else mod.chat
+ return make(
+ version="test",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=True,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+
+class FakeSyncStream:
+ """Sync context-manager stream shaped like groq's Stream."""
+
+ def __init__(self):
+ self._it = iter(CHUNKS)
+ self.closed = False
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self._it)
+
+ def close(self):
+ self.closed = True
+
+
+class FakeAsyncStream:
+ """Async context-manager stream shaped like groq's AsyncStream."""
+
+ def __init__(self):
+ self._it = iter(CHUNKS)
+ self.closed = False
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc):
+ return False
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self._it)
+ except StopIteration:
+ raise StopAsyncIteration from None
+
+ async def aclose(self):
+ self.closed = True
+
+
+async def _acreate(*_a, **_k):
+ """Async stand-in for groq's AsyncCompletions.create (awaitable)."""
+ return FakeAsyncStream()
+
+
+def _assert_one_span_with_tokens(exporter):
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1, "expected exactly one exported span"
+ attrs = spans[0].attributes
+ assert SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS in attrs
+ assert SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS in attrs
+
+
+def _assert_one_error_span(exporter):
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1, "expected exactly one exported span"
+ assert spans[0].status.status_code == StatusCode.ERROR
+ assert spans[0].attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"
+
+
+def test_sync_early_break_inside_with_ends_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ next(s) # consume one chunk, then leave the block early
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter)
+
+
+def test_sync_close_finalizes_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ next(s)
+ s.close()
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter)
+
+
+def test_sync_full_consumption_exports_exactly_one_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ for _ in s:
+ pass
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter)
+
+
+def test_sync_context_detached_after_early_break():
+ """The stream must not leave its span attached to the OTel context."""
+ tracer, _ = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ next(s)
+
+ assert trace_api.get_current_span(context_api.get_current()) is trace_api.INVALID_SPAN
+
+
+def test_sync_exception_inside_with_ends_error_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with pytest.raises(RuntimeError, match="stream failed"):
+ with stream as s:
+ next(s)
+ raise RuntimeError("stream failed")
+
+ _assert_one_error_span(exporter)
+
+
+async def test_async_early_break_inside_with_ends_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)
+ async with stream as s:
+ await anext(s)
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter)
+
+
+async def test_async_aclose_finalizes_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)
+ async with stream as s:
+ await anext(s)
+ await s.aclose()
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter)
+
+
+async def test_async_full_consumption_exports_exactly_one_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)
+ async with stream as s:
+ async for _ in s:
+ pass
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter)
+
+
+async def test_async_exception_inside_with_ends_error_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)
+ with pytest.raises(RuntimeError, match="stream failed"):
+ async with stream as s:
+ await anext(s)
+ raise RuntimeError("stream failed")
+
+ _assert_one_error_span(exporter)