Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(mistral): end streaming span on early break/close (#1517)
Paco Cartones433485c5aeaa
diff --git a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.pyindex 38592f29..dd6f0a46 100644--- a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py+++ b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py@@ -10,6 +10,7 @@ from openlit.__helpers import (set_server_address_and_port,record_completion_metrics,record_embedding_metrics,+ safe_detach,)from openlit.instrumentation.mistral.utils import (process_chunk,@@ -149,13 +150,25 @@ def async_stream(self._tbt = 0self._server_address = server_addressself._server_port = server_port+ self._streaming_response_processed = Falseasync def __aenter__(self):await self.__wrapped__.__aenter__()return selfasync def __aexit__(self, exc_type, exc_value, traceback):- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ try:+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ finally:+ if exc_type:+ self._streaming_response_processed = True+ handle_exception(self._span, exc_value)+ if self._span.is_recording():+ self._span.end()+ else:+ # A break before exhaustion never hits StopAsyncIteration,+ # so the span would leak without normal-exit finalization.+ self._finalize_streaming_span()def __aiter__(self):return self@@ -170,25 +183,46 @@ def async_stream(process_chunk(self, chunk)return chunkexcept StopAsyncIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=event_provider,- )-- except Exception as e:- handle_exception(self._span, e)-+ self._finalize_streaming_span()raise+ def _finalize_streaming_span(self):+ """Complete and end the span exactly once.++ Called on stream exhaustion and again from close()/manager exit;+ the flag keeps the double call a no-op so early exits that never+ see StopAsyncIteration still export the span instead of leaking it.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=event_provider,+ )++ except Exception as e:+ handle_exception(self._span, e)++ async def close(self):+ """Close the wrapped stream and finalize the span if not ended.++ Mistral's EventStreamAsync exposes ``close()``, not ``aclose()``.+ """+ try:+ await self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()+async def wrapper(wrapped, instance, args, kwargs):"""Wraps the GenAI stream function call.@@ -208,10 +242,10 @@ def async_stream(awaited_wrapped = await wrapped(*args, **kwargs)except Exception as e:handle_exception(span, e)- context_api.detach(token)+ safe_detach(token)span.end()raise- context_api.detach(token)+ safe_detach(token)return TracedAsyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_portdiff --git a/sdk/python/src/openlit/instrumentation/mistral/mistral.py b/sdk/python/src/openlit/instrumentation/mistral/mistral.pyindex dbbbe9bb..39085c51 100644--- a/sdk/python/src/openlit/instrumentation/mistral/mistral.py+++ b/sdk/python/src/openlit/instrumentation/mistral/mistral.py@@ -10,6 +10,7 @@ from openlit.__helpers import (set_server_address_and_port,record_completion_metrics,record_embedding_metrics,+ safe_detach,)from openlit.instrumentation.mistral.utils import (process_chunk,@@ -149,13 +150,25 @@ def stream(self._tbt = 0self._server_address = server_addressself._server_port = server_port+ self._streaming_response_processed = Falsedef __enter__(self):self.__wrapped__.__enter__()return selfdef __exit__(self, exc_type, exc_value, traceback):- self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ try:+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ finally:+ if exc_type:+ self._streaming_response_processed = True+ handle_exception(self._span, exc_value)+ if self._span.is_recording():+ self._span.end()+ else:+ # A break before exhaustion never hits StopIteration, so+ # the span would leak without normal-exit finalization.+ self._finalize_streaming_span()def __iter__(self):return self@@ -170,25 +183,43 @@ def stream(process_chunk(self, chunk)return chunkexcept StopIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=event_provider,- )-- except Exception as e:- handle_exception(self._span, e)-+ self._finalize_streaming_span()raise+ def _finalize_streaming_span(self):+ """Complete and end the span exactly once.++ Called on stream exhaustion and again from close()/manager exit;+ the flag keeps the double call a no-op so early exits that never+ see StopIteration still export the span instead of leaking it.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=event_provider,+ )++ except Exception as e:+ handle_exception(self._span, e)++ def close(self):+ """Close the wrapped stream and finalize the span if not ended."""+ try:+ self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()+def wrapper(wrapped, instance, args, kwargs):"""Wraps the GenAI stream function call.@@ -208,10 +239,10 @@ def stream(awaited_wrapped = wrapped(*args, **kwargs)except Exception as e:handle_exception(span, e)- context_api.detach(token)+ safe_detach(token)span.end()raise- context_api.detach(token)+ safe_detach(token)return TracedSyncStream(awaited_wrapped, span, span_name, kwargs, server_address, server_portdiff --git a/sdk/python/tests/test_mistral_stream_span_lifecycle.py b/sdk/python/tests/test_mistral_stream_span_lifecycle.pynew file mode 100644index 00000000..7b38553f--- /dev/null+++ b/sdk/python/tests/test_mistral_stream_span_lifecycle.py@@ -0,0 +1,265 @@+# pylint: disable=protected-access, duplicate-code, missing-function-docstring+"""Regression tests: the Mistral streaming wrappers must end their span on+every exit path.++`TracedSyncStream.__exit__`/`TracedAsyncStream.__aexit__` merely forwarded to+the wrapped stream and there was no `close()`, so the span was ended only+inside the `except StopIteration`/`StopAsyncIteration` handler. A caller that+`break`s out of `with … as stream:` before the stream is exhausted, or calls+`stream.close()` early, never hit that handler, so the span stayed recording+forever and was never exported — the whole call (span, cost, tokens) was+lost. This is the Mistral instance of the early-close streaming-span leak+fixed for Anthropic in #1461 and Groq in #1516 (distinct from the async+postflight bug #1495/#1496).++These tests drive the real `stream`/`async_stream` wrapper factories with a+synthetic Mistral-shaped event stream and assert the span ends exactly once,+carrying token-usage attributes, on each exit path.+"""++import time++import pytest+from opentelemetry import trace as trace_api, context as context_api+from opentelemetry.trace import StatusCode+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++from openlit._config import OpenlitConfig+from openlit.instrumentation.mistral import mistral as sync_mod+from openlit.instrumentation.mistral import async_mistral as async_mod+from openlit.semcov import SemanticConvention++REQUEST_KWARGS = {+ "model": "mistral-small-latest",+ "messages": [{"role": "user", "content": "Monitor LLM Applications"}],+}++# Two Mistral-shaped events: a content delta, then a final event with usage.+CHUNKS = [+ {"data": {"choices": [{"delta": {"content": "partial"}, "finish_reason": None}]}},+ {+ "data": {+ "id": "cmpl-1",+ "model": "mistral-small-latest",+ "choices": [{"delta": {"content": " answer"}, "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 10, "completion_tokens": 5},+ }+ },+]+++def _tracer_with_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _factory(tracer, *, is_async):+ mod = async_mod if is_async else sync_mod+ make = mod.async_stream if is_async else mod.stream+ return make(+ version="test",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=True,+ metrics=None,+ disable_metrics=True,+ event_provider=None,+ )+++class FakeSyncStream:+ """Sync context-manager stream shaped like mistral's event stream."""++ def __init__(self):+ self._it = iter(CHUNKS)+ self.closed = False++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False++ def __iter__(self):+ return self++ def __next__(self):+ return next(self._it)++ def close(self):+ self.closed = True+++class FakeAsyncStream:+ """Async context-manager stream shaped like mistral's event stream."""++ def __init__(self):+ self._it = iter(CHUNKS)+ self.closed = False++ async def __aenter__(self):+ return self++ async def __aexit__(self, *exc):+ return False++ def __aiter__(self):+ return self++ async def __anext__(self):+ try:+ return next(self._it)+ except StopIteration:+ raise StopAsyncIteration from None++ async def close(self):+ self.closed = True+++async def _acreate(*_a, **_k):+ """Async stand-in for mistral's chat.stream_async (awaitable)."""+ return FakeAsyncStream()+++def _assert_one_span_with_tokens(exporter, *, input_tokens, output_tokens):+ spans = exporter.get_finished_spans()+ assert len(spans) == 1, "expected exactly one exported span"+ attrs = spans[0].attributes+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == input_tokens+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == output_tokens+++def _assert_one_error_span(exporter):+ spans = exporter.get_finished_spans()+ assert len(spans) == 1, "expected exactly one exported span"+ assert spans[0].status.status_code == StatusCode.ERROR+ assert spans[0].attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"+++def test_sync_early_break_inside_with_ends_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ next(s) # consume one event, then leave the block early++ time.sleep(0.05)+ # The final usage-bearing event was not consumed, so an early exit must+ # export the span without inventing token counts.+ _assert_one_span_with_tokens(exporter, input_tokens=0, output_tokens=0)+++def test_sync_close_finalizes_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ next(s)+ s.close()++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter, input_tokens=0, output_tokens=0)+++def test_sync_full_consumption_exports_exactly_one_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ for _ in s:+ pass++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter, input_tokens=10, output_tokens=5)+++def test_sync_context_detached_after_early_break():+ """The stream must not leave its span attached to the OTel context."""+ tracer, _ = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ next(s)++ assert trace_api.get_current_span(context_api.get_current()) is trace_api.INVALID_SPAN+++def test_sync_exception_inside_with_ends_error_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=False)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with pytest.raises(RuntimeError, match="stream failed"):+ with stream as s:+ next(s)+ raise RuntimeError("stream failed")++ _assert_one_error_span(exporter)+++async def test_async_early_break_inside_with_ends_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=True)++ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)+ async with stream as s:+ await anext(s)++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter, input_tokens=0, output_tokens=0)+++async def test_async_close_finalizes_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=True)++ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)+ async with stream as s:+ await anext(s)+ await s.close()++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter, input_tokens=0, output_tokens=0)+++async def test_async_full_consumption_exports_exactly_one_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=True)++ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)+ async with stream as s:+ async for _ in s:+ pass++ time.sleep(0.05)+ _assert_one_span_with_tokens(exporter, input_tokens=10, output_tokens=5)+++async def test_async_exception_inside_with_ends_error_span():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer, is_async=True)++ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)+ with pytest.raises(RuntimeError, match="stream failed"):+ async with stream as s:+ await anext(s)+ raise RuntimeError("stream failed")++ _assert_one_error_span(exporter)