Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(mistral): end streaming span on early break/close (#1517)

Paco Cartones433485c5aeaa
diff --git a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
index 38592f29..dd6f0a46 100644
--- a/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
+++ b/sdk/python/src/openlit/instrumentation/mistral/async_mistral.py
@@ -10,6 +10,7 @@ from openlit.__helpers import (
set_server_address_and_port,
record_completion_metrics,
record_embedding_metrics,
+ safe_detach,
)
from openlit.instrumentation.mistral.utils import (
process_chunk,
@@ -149,13 +150,25 @@ def async_stream(
self._tbt = 0
self._server_address = server_address
self._server_port = server_port
+ self._streaming_response_processed = False
async def __aenter__(self):
await self.__wrapped__.__aenter__()
return self
async def __aexit__(self, exc_type, exc_value, traceback):
- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+ try:
+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+ finally:
+ if exc_type:
+ self._streaming_response_processed = True
+ handle_exception(self._span, exc_value)
+ if self._span.is_recording():
+ self._span.end()
+ else:
+ # A break before exhaustion never hits StopAsyncIteration,
+ # so the span would leak without normal-exit finalization.
+ self._finalize_streaming_span()
def __aiter__(self):
return self
@@ -170,25 +183,46 @@ def async_stream(
process_chunk(self, chunk)
return chunk
except StopAsyncIteration:
- try:
- with self._span:
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=event_provider,
- )
-
- except Exception as e:
- handle_exception(self._span, e)
-
+ self._finalize_streaming_span()
raise
+ def _finalize_streaming_span(self):
+ """Complete and end the span exactly once.
+
+ Called on stream exhaustion and again from close()/manager exit;
+ the flag keeps the double call a no-op so early exits that never
+ see StopAsyncIteration still export the span instead of leaking it.
+ """
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ try:
+ with self._span:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=event_provider,
+ )
+
+ except Exception as e:
+ handle_exception(self._span, e)
+
+ async def close(self):
+ """Close the wrapped stream and finalize the span if not ended.
+
+ Mistral's EventStreamAsync exposes ``close()``, not ``aclose()``.
+ """
+ try:
+ await self.__wrapped__.close()
+ finally:
+ self._finalize_streaming_span()
+
async def wrapper(wrapped, instance, args, kwargs):
"""
Wraps the GenAI stream function call.
@@ -208,10 +242,10 @@ def async_stream(
awaited_wrapped = await wrapped(*args, **kwargs)
except Exception as e:
handle_exception(span, e)
- context_api.detach(token)
+ safe_detach(token)
span.end()
raise
- context_api.detach(token)
+ safe_detach(token)
return TracedAsyncStream(
awaited_wrapped, span, span_name, kwargs, server_address, server_port
diff --git a/sdk/python/src/openlit/instrumentation/mistral/mistral.py b/sdk/python/src/openlit/instrumentation/mistral/mistral.py
index dbbbe9bb..39085c51 100644
--- a/sdk/python/src/openlit/instrumentation/mistral/mistral.py
+++ b/sdk/python/src/openlit/instrumentation/mistral/mistral.py
@@ -10,6 +10,7 @@ from openlit.__helpers import (
set_server_address_and_port,
record_completion_metrics,
record_embedding_metrics,
+ safe_detach,
)
from openlit.instrumentation.mistral.utils import (
process_chunk,
@@ -149,13 +150,25 @@ def stream(
self._tbt = 0
self._server_address = server_address
self._server_port = server_port
+ self._streaming_response_processed = False
def __enter__(self):
self.__wrapped__.__enter__()
return self
def __exit__(self, exc_type, exc_value, traceback):
- self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+ try:
+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+ finally:
+ if exc_type:
+ self._streaming_response_processed = True
+ handle_exception(self._span, exc_value)
+ if self._span.is_recording():
+ self._span.end()
+ else:
+ # A break before exhaustion never hits StopIteration, so
+ # the span would leak without normal-exit finalization.
+ self._finalize_streaming_span()
def __iter__(self):
return self
@@ -170,25 +183,43 @@ def stream(
process_chunk(self, chunk)
return chunk
except StopIteration:
- try:
- with self._span:
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=event_provider,
- )
-
- except Exception as e:
- handle_exception(self._span, e)
-
+ self._finalize_streaming_span()
raise
+ def _finalize_streaming_span(self):
+ """Complete and end the span exactly once.
+
+ Called on stream exhaustion and again from close()/manager exit;
+ the flag keeps the double call a no-op so early exits that never
+ see StopIteration still export the span instead of leaking it.
+ """
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ try:
+ with self._span:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=event_provider,
+ )
+
+ except Exception as e:
+ handle_exception(self._span, e)
+
+ def close(self):
+ """Close the wrapped stream and finalize the span if not ended."""
+ try:
+ self.__wrapped__.close()
+ finally:
+ self._finalize_streaming_span()
+
def wrapper(wrapped, instance, args, kwargs):
"""
Wraps the GenAI stream function call.
@@ -208,10 +239,10 @@ def stream(
awaited_wrapped = wrapped(*args, **kwargs)
except Exception as e:
handle_exception(span, e)
- context_api.detach(token)
+ safe_detach(token)
span.end()
raise
- context_api.detach(token)
+ safe_detach(token)
return TracedSyncStream(
awaited_wrapped, span, span_name, kwargs, server_address, server_port
diff --git a/sdk/python/tests/test_mistral_stream_span_lifecycle.py b/sdk/python/tests/test_mistral_stream_span_lifecycle.py
new file mode 100644
index 00000000..7b38553f
--- /dev/null
+++ b/sdk/python/tests/test_mistral_stream_span_lifecycle.py
@@ -0,0 +1,265 @@
+# pylint: disable=protected-access, duplicate-code, missing-function-docstring
+"""Regression tests: the Mistral streaming wrappers must end their span on
+every exit path.
+
+`TracedSyncStream.__exit__`/`TracedAsyncStream.__aexit__` merely forwarded to
+the wrapped stream and there was no `close()`, so the span was ended only
+inside the `except StopIteration`/`StopAsyncIteration` handler. A caller that
+`break`s out of `with … as stream:` before the stream is exhausted, or calls
+`stream.close()` early, never hit that handler, so the span stayed recording
+forever and was never exported — the whole call (span, cost, tokens) was
+lost. This is the Mistral instance of the early-close streaming-span leak
+fixed for Anthropic in #1461 and Groq in #1516 (distinct from the async
+postflight bug #1495/#1496).
+
+These tests drive the real `stream`/`async_stream` wrapper factories with a
+synthetic Mistral-shaped event stream and assert the span ends exactly once,
+carrying token-usage attributes, on each exit path.
+"""
+
+import time
+
+import pytest
+from opentelemetry import trace as trace_api, context as context_api
+from opentelemetry.trace import StatusCode
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.mistral import mistral as sync_mod
+from openlit.instrumentation.mistral import async_mistral as async_mod
+from openlit.semcov import SemanticConvention
+
+REQUEST_KWARGS = {
+ "model": "mistral-small-latest",
+ "messages": [{"role": "user", "content": "Monitor LLM Applications"}],
+}
+
+# Two Mistral-shaped events: a content delta, then a final event with usage.
+CHUNKS = [
+ {"data": {"choices": [{"delta": {"content": "partial"}, "finish_reason": None}]}},
+ {
+ "data": {
+ "id": "cmpl-1",
+ "model": "mistral-small-latest",
+ "choices": [{"delta": {"content": " answer"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5},
+ }
+ },
+]
+
+
+def _tracer_with_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return provider.get_tracer(__name__), exporter
+
+
+def _factory(tracer, *, is_async):
+ mod = async_mod if is_async else sync_mod
+ make = mod.async_stream if is_async else mod.stream
+ return make(
+ version="test",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=True,
+ metrics=None,
+ disable_metrics=True,
+ event_provider=None,
+ )
+
+
+class FakeSyncStream:
+ """Sync context-manager stream shaped like mistral's event stream."""
+
+ def __init__(self):
+ self._it = iter(CHUNKS)
+ self.closed = False
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self._it)
+
+ def close(self):
+ self.closed = True
+
+
+class FakeAsyncStream:
+ """Async context-manager stream shaped like mistral's event stream."""
+
+ def __init__(self):
+ self._it = iter(CHUNKS)
+ self.closed = False
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc):
+ return False
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ try:
+ return next(self._it)
+ except StopIteration:
+ raise StopAsyncIteration from None
+
+ async def close(self):
+ self.closed = True
+
+
+async def _acreate(*_a, **_k):
+ """Async stand-in for mistral's chat.stream_async (awaitable)."""
+ return FakeAsyncStream()
+
+
+def _assert_one_span_with_tokens(exporter, *, input_tokens, output_tokens):
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1, "expected exactly one exported span"
+ attrs = spans[0].attributes
+ assert attrs[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == input_tokens
+ assert attrs[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == output_tokens
+
+
+def _assert_one_error_span(exporter):
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1, "expected exactly one exported span"
+ assert spans[0].status.status_code == StatusCode.ERROR
+ assert spans[0].attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"
+
+
+def test_sync_early_break_inside_with_ends_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ next(s) # consume one event, then leave the block early
+
+ time.sleep(0.05)
+ # The final usage-bearing event was not consumed, so an early exit must
+ # export the span without inventing token counts.
+ _assert_one_span_with_tokens(exporter, input_tokens=0, output_tokens=0)
+
+
+def test_sync_close_finalizes_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ next(s)
+ s.close()
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter, input_tokens=0, output_tokens=0)
+
+
+def test_sync_full_consumption_exports_exactly_one_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ for _ in s:
+ pass
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter, input_tokens=10, output_tokens=5)
+
+
+def test_sync_context_detached_after_early_break():
+ """The stream must not leave its span attached to the OTel context."""
+ tracer, _ = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ next(s)
+
+ assert trace_api.get_current_span(context_api.get_current()) is trace_api.INVALID_SPAN
+
+
+def test_sync_exception_inside_with_ends_error_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with pytest.raises(RuntimeError, match="stream failed"):
+ with stream as s:
+ next(s)
+ raise RuntimeError("stream failed")
+
+ _assert_one_error_span(exporter)
+
+
+async def test_async_early_break_inside_with_ends_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)
+ async with stream as s:
+ await anext(s)
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter, input_tokens=0, output_tokens=0)
+
+
+async def test_async_close_finalizes_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)
+ async with stream as s:
+ await anext(s)
+ await s.close()
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter, input_tokens=0, output_tokens=0)
+
+
+async def test_async_full_consumption_exports_exactly_one_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)
+ async with stream as s:
+ async for _ in s:
+ pass
+
+ time.sleep(0.05)
+ _assert_one_span_with_tokens(exporter, input_tokens=10, output_tokens=5)
+
+
+async def test_async_exception_inside_with_ends_error_span():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(_acreate, None, (), REQUEST_KWARGS)
+ with pytest.raises(RuntimeError, match="stream failed"):
+ async with stream as s:
+ await anext(s)
+ raise RuntimeError("stream failed")
+
+ _assert_one_error_span(exporter)