Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(groq): end streaming span on early break/close (#1516)
Paco Cartones864a553b09a2
diff --git a/sdk/python/src/openlit/instrumentation/groq/async_groq.py b/sdk/python/src/openlit/instrumentation/groq/async_groq.pyindex 686f4353..562f8199 100644--- a/sdk/python/src/openlit/instrumentation/groq/async_groq.py+++ b/sdk/python/src/openlit/instrumentation/groq/async_groq.py@@ -9,6 +9,7 @@ from openlit.__helpers import (handle_exception,set_server_address_and_port,record_completion_metrics,+ safe_detach,)from openlit.instrumentation.groq.utils import (process_chunk,@@ -72,13 +73,25 @@ def async_chat(self._tbt = 0self._server_address = server_addressself._server_port = server_port+ self._streaming_response_processed = Falseasync def __aenter__(self):await self.__wrapped__.__aenter__()return selfasync def __aexit__(self, exc_type, exc_value, traceback):- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ try:+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ finally:+ if exc_type:+ self._streaming_response_processed = True+ handle_exception(self._span, exc_value)+ if self._span.is_recording():+ self._span.end()+ else:+ # A break before exhaustion never hits StopAsyncIteration,+ # so the span would leak without normal-exit finalization.+ self._finalize_streaming_span()def __aiter__(self):return self@@ -93,24 +106,42 @@ def async_chat(process_chunk(self, chunk)return chunkexcept StopAsyncIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=event_provider,- )+ self._finalize_streaming_span()+ raise- except Exception as e:- handle_exception(self._span, e)+ def _finalize_streaming_span(self):+ """Complete and end the span exactly once.- raise+ Called on stream exhaustion and again from aclose()/manager exit;+ the flag keeps the double call a no-op so early exits that never+ see StopAsyncIteration still export the span instead of leaking it.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=event_provider,+ )++ except Exception as e:+ handle_exception(self._span, e)++ async def aclose(self):+ """Close the wrapped stream and finalize the span if not ended."""+ try:+ await self.__wrapped__.aclose()+ finally:+ self._finalize_streaming_span()async def wrapper(wrapped, instance, args, kwargs):"""@@ -133,10 +164,10 @@ def async_chat(awaited_wrapped = await wrapped(*args, **kwargs)except Exception as e:handle_exception(span, e)- context_api.detach(token)+ safe_detach(token)span.end()raise- context_api.detach(token)+ safe_detach(token)return TracedAsyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)diff --git a/sdk/python/src/openlit/instrumentation/groq/groq.py b/sdk/python/src/openlit/instrumentation/groq/groq.pyindex e9492937..e89d413c 100644--- a/sdk/python/src/openlit/instrumentation/groq/groq.py+++ b/sdk/python/src/openlit/instrumentation/groq/groq.py@@ -9,6 +9,7 @@ from openlit.__helpers import (handle_exception,set_server_address_and_port,record_completion_metrics,+ safe_detach,)from openlit.instrumentation.groq.utils import (process_chunk,@@ -72,13 +73,25 @@ def chat(self._tbt = 0self._server_address = server_addressself._server_port = server_port+ self._streaming_response_processed = Falsedef __enter__(self):self.__wrapped__.__enter__()return selfdef __exit__(self, exc_type, exc_value, traceback):- self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ try:+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ finally:+ if exc_type:+ self._streaming_response_processed = True+ handle_exception(self._span, exc_value)+ if self._span.is_recording():+ self._span.end()+ else:+ # A break before exhaustion never hits StopIteration, so+ # the span would leak without normal-exit finalization.+ self._finalize_streaming_span()def __iter__(self):return self@@ -93,24 +106,42 @@ def chat(process_chunk(self, chunk)return chunkexcept StopIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=event_provider,- )+ self._finalize_streaming_span()+ raise- except Exception as e:- handle_exception(self._span, e)+ def _finalize_streaming_span(self):+ """Complete and end the span exactly once.- raise+ Called on stream exhaustion and again from close()/manager exit;+ the flag keeps the double call a no-op so early exits that never+ see StopIteration still export the span instead of leaking it.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=event_provider,+ )++ except Exception as e:+ handle_exception(self._span, e)++ def close(self):+ """Close the wrapped stream and finalize the span if not ended."""+ try:+ self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()def wrapper(wrapped, instance, args, kwargs):"""@@ -133,10 +164,10 @@ def chat(awaited_wrapped = wrapped(*args, **kwargs)except Exception as e:handle_exception(span, e)- context_api.detach(token)+ safe_detach(token)span.end()raise- context_api.detach(token)+ safe_detach(token)return TracedSyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_port)diff --git a/sdk/python/tests/test_groq_stream_span_lifecycle.py b/sdk/python/tests/test_groq_stream_span_lifecycle.pynew file mode 100644index 00000000..f0f8dc0c--- /dev/null+++ b/sdk/python/tests/test_groq_stream_span_lifecycle.py@@ -0,0 +1,264 @@+# pylint: disable=protected-access, duplicate-code, missing-function-docstring+"""Regression tests: the Groq streaming wrappers must end their span on every+exit path.++`TracedSyncStream.__exit__`/`TracedAsyncStream.__aexit__` merely forwarded to+the wrapped stream and there was no `close()`/`aclose()`, so the span was+ended only inside the `except StopIteration`/`StopAsyncIteration` handler.+A caller that `break`s out of `with … as stream:` before the stream is+exhausted, or calls `stream.close()` early, never hit that handler, so the+span stayed recording forever and was never exported — the whole call+(span, cost, tokens) was lost. This is the Groq instance of the early-close+streaming-span leak fixed for Anthropic in #1461 and filed for OpenAI in+#1454/#1455.++These tests drive the real `chat`/`async_chat` wrapper factories with a+synthetic Groq-shaped stream and assert the span ends exactly once, carrying+token-usage attributes, on each exit path.+"""++import time++import pytest+from opentelemetry import trace as trace_api, context as context_api+from opentelemetry.trace import StatusCode+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++from openlit._config import OpenlitConfig+from openlit.instrumentation.groq import groq as sync_mod+from openlit.instrumentation.groq import async_groq as async_mod+from openlit.semcov import SemanticConvention++REQUEST_KWARGS = {+ "model": "llama-3.1-8b-instant",+ "stream": True,+ "messages": [{"role": "user", "content": "Monitor LLM Applications"}],+}++# Two Groq-shaped chunks: a content delta, then a final chunk carrying usage.+CHUNKS = [+ {"choices": [{"delta": {"content": "partial"}, "finish_reason": None}]},+ {+ "choices": [{"delta": {"content": " answer"}, "finish_reason": "stop"}],+ "x_groq": {+ "id": "chatcmpl-1",+ "model": "llama-3.1-8b-instant",+ "system_fingerprint": "fp_test",+ "usage": {"prompt_tokens": 10, "completion_tokens": 5},+ },+ },+]+++def _tracer_with_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _factory(tracer, *, is_async):+ mod = async_mod if is_async else sync_mod+ make = mod.async_chat if is_async else mod.chat+ return make(+ version="test",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=True,+ metrics=None,+ disable_metrics=True,+ )+++class FakeSyncStream:+ """Sync context-manager stream shaped like groq's Stream."""++ def __init__(self):+ self._it = iter(CHUNKS)+ self.closed = False++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False++ def __iter__(self):+ return self++ def __next__(self):+ return next(self._it)++ def close(self):+ self.closed = True+++class FakeAsyncStream:+ """Async context-manager stream shaped like groq's AsyncStream."""++ def __init__(self):+ self._it = iter(CHUNKS)+ self.closed = False++ async def __aenter__(self):+ return self++ async def __aexit__(self, *exc):+ return False++ def __aiter__(self):+ return self++ async def __anext__(self):+ try:+ return next(self._it)+ except StopIteration:+ raise StopAsyncIteration from None++ async def aclose(self):+ self.closed = True+++async def _acreate(*_a, **_k):+ """Async stand-in for groq's AsyncCompletions.create (awaitable)."""+ return FakeAsyncStream()+++def _assert_one_span_with_tokens(exporter):+ spans = exporter.get_finished_spans()+ assert len(spans) == 1, "expected exactly one exported span"+ attrs = spans[0].attributes+ assert SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS in attrs+ assert SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS in attrs+++def _assert_one_error_span(exporter):+ spans = exporter.get_finished_spans()+ assert len(spans) == 1, "expected exactly one exported span"+ assert spans[0].status.status_code == StatusCode.ERROR+ assert spans[0].attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"+++def test_sync_early_break_inside_with_ends_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ next(s) # consume one chunk, then leave the block early++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter)+++def test_sync_close_finalizes_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ next(s)+ s.close()++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter)+++def test_sync_full_consumption_exports_exactly_one_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ for _ in s:+ pass++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter)+++def test_sync_context_detached_after_early_break():+ """The stream must not leave its span attached to the OTel context."""+ tracer, _ = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ next(s)++ assert trace_api.get_current_span(context_api.get_current()) is trace_api.INVALID_SPAN+++def test_sync_exception_inside_with_ends_error_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with pytest.raises(RuntimeError, match="stream failed"):+ with stream as s:+ next(s)+ raise RuntimeError("stream failed")++ _assert_one_error_span(exporter)+++async def test_async_early_break_inside_with_ends_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=True)++ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)+ async with stream as s:+ await anext(s)++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter)+++async def test_async_aclose_finalizes_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=True)++ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)+ async with stream as s:+ await anext(s)+ await s.aclose()++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter)+++async def test_async_full_consumption_exports_exactly_one_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=True)++ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)+ async with stream as s:+ async for _ in s:+ pass++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter)+++async def test_async_exception_inside_with_ends_error_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=True)++ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)+ with pytest.raises(RuntimeError, match="stream failed"):+ async with stream as s:+ await anext(s)+ raise RuntimeError("stream failed")++ _assert_one_error_span(exporter)