Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(litellm): end streaming spans on early exit (#1616)

RichardoMu2a7d01d0cf1f
diff --git a/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py b/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
index ef158dc0..39a8ff58 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/async_litellm.py
@@ -2,8 +2,9 @@
Module for monitoring LiteLLM API calls (async version).
"""
-import time
import asyncio
+import inspect
+import time
from opentelemetry import trace as trace_api, context as context_api
from opentelemetry.trace import SpanKind
from openlit.__helpers import (
@@ -41,7 +42,7 @@ def acompletion(
Generates a telemetry wrapper for GenAI function call
"""
- class TracedAsyncStream:
+ class TracedAsyncStream: # pylint: disable=too-many-instance-attributes
"""
Wrapper for async streaming responses to collect telemetry.
"""
@@ -60,6 +61,7 @@ def acompletion(
self.__wrapped__ = wrapped
self._span = span
self._span_name = span_name
+ self._streaming_response_processed = False
self._llmresponse = ""
self._response_id = ""
self._response_model = ""
@@ -81,13 +83,30 @@ def acompletion(
self._server_address = server_address
self._server_port = server_port
self._event_provider = event_provider
+ self._stream_error = None
async def __aenter__(self):
- await self.__wrapped__.__aenter__()
+ # CustomStreamWrapper.__aenter__(self) exists only on 1.107.
+ enter = getattr(self.__wrapped__, "__aenter__", None)
+ if enter is not None:
+ await enter()
return self
async def __aexit__(self, exc_type, exc_value, traceback):
- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+ # 1.107: async def __aexit__(self, *exc_info) -> None awaits aclose().
+ # 1.84 through 1.105.1 define async aclose(self) and no __aexit__.
+ exit_fn = getattr(self.__wrapped__, "__aexit__", None)
+ if exit_fn is None:
+ await self.aclose()
+ return None
+ try:
+ return await exit_fn(exc_type, exc_value, traceback)
+ except BaseException as exit_exc:
+ if self._stream_error is None:
+ self._stream_error = exit_exc
+ raise
+ finally:
+ self._finalize_streaming_span(ended_cleanly=False)
def __aiter__(self):
return self
@@ -96,56 +115,108 @@ def acompletion(
"""Delegate attribute access to the wrapped object."""
return getattr(self.__wrapped__, name)
+ def __del__(self):
+ """Finalize a span abandoned by a bare ``async for`` loop.
+
+ CustomStreamWrapper is not an async generator, so leaving the loop
+ does not schedule ``aclose()``. Dealloc cannot await; HTTP cleanup
+ stays with LiteLLM.
+ """
+ try:
+ self._finalize_streaming_span(ended_cleanly=False)
+ except Exception:
+ pass
+
async def __anext__(self):
try:
chunk = await self.__wrapped__.__anext__()
process_chunk(self, chunk)
return chunk
except StopAsyncIteration:
- try:
- # Use the existing span that was started when the stream began
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=self._event_provider,
- )
- # End the span after processing
- self._span.end()
-
- except Exception as e:
- handle_exception(self._span, e)
- self._span.end()
-
+ self._finalize_streaming_span(ended_cleanly=True)
raise
except (GeneratorExit, asyncio.CancelledError):
- try:
- # Use the existing span that was started when the stream began
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=self._event_provider,
- )
- # End the span after processing
- self._span.end()
-
- except Exception as e:
- handle_exception(self._span, e)
- self._span.end()
-
+ self._finalize_streaming_span(ended_cleanly=False)
+ raise
+ except Exception as exc:
+ self._stream_error = exc
raise
+ def _finalize_streaming_span(self, ended_cleanly):
+ """End the span once.
+
+ ``ended_cleanly`` is True only for StopAsyncIteration. An empty
+ finish reason is otherwise a cancel. A failure raised by the
+ wrapped stream is recorded after attributes are set, so partial
+ text stays on the span and the status is ERROR rather than the OK
+ that ``process_streaming_chat_response`` writes when nothing failed.
+ """
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ if self._stream_error is not None and not self._finish_reason:
+ self._finish_reason = "error"
+ elif not ended_cleanly and not self._finish_reason:
+ self._finish_reason = "incomplete"
+ try:
+ with self._span:
+ try:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=self._event_provider,
+ )
+ finally:
+ if self._stream_error is not None:
+ handle_exception(self._span, self._stream_error)
+ except Exception as e:
+ handle_exception(self._span, e)
+
+ def close(self):
+ """Call sync ``CustomStreamWrapper.close(self) -> None`` and finalize.
+
+ On 1.107 the async class has this synchronous method beside
+ ``async def aclose(self)``. The two are not aliases: ``close``
+ returns None and must not be a coroutine.
+ """
+ try:
+ closer = getattr(self.__wrapped__, "close", None)
+ if closer is not None:
+ try:
+ result = closer()
+ if inspect.iscoroutine(result):
+ result.close()
+ except BaseException as exit_exc:
+ if self._stream_error is None:
+ self._stream_error = exit_exc
+ raise
+ finally:
+ self._finalize_streaming_span(ended_cleanly=False)
+
+ async def aclose(self):
+ """Await ``CustomStreamWrapper.aclose(self)`` and finalize.
+
+ ``aclose`` has shipped since the 1.84 line. On 1.107,
+ ``__aexit__`` is ``await self.aclose()``.
+ """
+ try:
+ closer = getattr(self.__wrapped__, "aclose", None)
+ if closer is not None:
+ try:
+ await closer()
+ except BaseException as exit_exc:
+ if self._stream_error is None:
+ self._stream_error = exit_exc
+ raise
+ finally:
+ self._finalize_streaming_span(ended_cleanly=False)
+
async def wrapper(wrapped, instance, args, kwargs):
"""
Wraps the GenAI function call.
diff --git a/sdk/python/src/openlit/instrumentation/litellm/litellm.py b/sdk/python/src/openlit/instrumentation/litellm/litellm.py
index 830d22de..e087feec 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/litellm.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/litellm.py
@@ -2,6 +2,7 @@
Module for monitoring LiteLLM API calls.
"""
+import inspect
import time
from opentelemetry import trace as trace_api, context as context_api
from opentelemetry.trace import SpanKind
@@ -40,7 +41,7 @@ def completion(
Generates a telemetry wrapper for GenAI function call
"""
- class TracedSyncStream:
+ class TracedSyncStream: # pylint: disable=too-many-instance-attributes
"""
Wrapper for streaming responses to collect telemetry.
"""
@@ -59,6 +60,7 @@ def completion(
self.__wrapped__ = wrapped
self._span = span
self._span_name = span_name
+ self._streaming_response_processed = False
self._llmresponse = ""
self._response_id = ""
self._response_model = ""
@@ -80,13 +82,31 @@ def completion(
self._server_address = server_address
self._server_port = server_port
self._event_provider = event_provider
+ self._stream_error = None
def __enter__(self):
- self.__wrapped__.__enter__()
+ # CustomStreamWrapper.__enter__(self) -> CustomStreamWrapper exists
+ # only on unreleased 1.107. Releases through 1.105.1 are iterators.
+ enter = getattr(self.__wrapped__, "__enter__", None)
+ if enter is not None:
+ enter()
return self
def __exit__(self, exc_type, exc_value, traceback):
- self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+ # 1.107 defines __exit__(self, *exc_info) -> None as self.close().
+ # Older releases have no __exit__; close() finalizes on its own.
+ exit_fn = getattr(self.__wrapped__, "__exit__", None)
+ if exit_fn is None:
+ self.close()
+ return None
+ try:
+ return exit_fn(exc_type, exc_value, traceback)
+ except BaseException as exit_exc:
+ if self._stream_error is None:
+ self._stream_error = exit_exc
+ raise
+ finally:
+ self._finalize_streaming_span(ended_cleanly=False)
def __iter__(self):
return self
@@ -95,33 +115,87 @@ def completion(
"""Delegate attribute access to the wrapped object."""
return getattr(self.__wrapped__, name)
+ def __del__(self):
+ """Finalize a span abandoned by a bare ``for`` loop.
+
+ CustomStreamWrapper is a class iterator, so ``break`` calls neither
+ ``close()`` nor ``__exit__``. Releases through 1.105.1 have neither
+ method. Dealloc only ends the span; HTTP cleanup stays with LiteLLM.
+ """
+ try:
+ self._finalize_streaming_span(ended_cleanly=False)
+ except Exception:
+ pass
+
def __next__(self):
try:
chunk = self.__wrapped__.__next__()
process_chunk(self, chunk)
return chunk
except StopIteration:
- try:
- # Use the existing span that was started when the stream began
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=self._event_provider,
- )
- # End the span after processing
- self._span.end()
+ self._finalize_streaming_span(ended_cleanly=True)
+ raise
+ except Exception as exc:
+ self._stream_error = exc
+ raise
- except Exception as e:
- handle_exception(self._span, e)
- self._span.end()
+ def _finalize_streaming_span(self, ended_cleanly):
+ """End the span once.
- raise
+ ``ended_cleanly`` is True only for StopIteration. An empty finish
+ reason is otherwise a cancel. A failure raised by the wrapped
+ stream is recorded after attributes are set, so partial text stays
+ on the span and the status is ERROR rather than the OK that
+ ``process_streaming_chat_response`` writes when nothing failed.
+ """
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ if self._stream_error is not None and not self._finish_reason:
+ self._finish_reason = "error"
+ elif not ended_cleanly and not self._finish_reason:
+ self._finish_reason = "incomplete"
+ try:
+ with self._span:
+ try:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=self._event_provider,
+ )
+ finally:
+ if self._stream_error is not None:
+ handle_exception(self._span, self._stream_error)
+ except Exception as e:
+ handle_exception(self._span, e)
+
+ def close(self):
+ """Call ``CustomStreamWrapper.close(self) -> None`` and finalize.
+
+ That method is synchronous and ships on 1.107. Releases through
+ 1.105.1 have no ``close``; the span is still finalized.
+ """
+ try:
+ closer = getattr(self.__wrapped__, "close", None)
+ if closer is not None:
+ try:
+ result = closer()
+ # close() returns None. Drop a coroutine from a
+ # mismatched object so it cannot warn "never awaited".
+ if inspect.iscoroutine(result):
+ result.close()
+ except BaseException as exit_exc:
+ if self._stream_error is None:
+ self._stream_error = exit_exc
+ raise
+ finally:
+ self._finalize_streaming_span(ended_cleanly=False)
def wrapper(wrapped, instance, args, kwargs):
"""
diff --git a/sdk/python/src/openlit/instrumentation/litellm/utils.py b/sdk/python/src/openlit/instrumentation/litellm/utils.py
index 54d0bad0..ebc0ec26 100644
--- a/sdk/python/src/openlit/instrumentation/litellm/utils.py
+++ b/sdk/python/src/openlit/instrumentation/litellm/utils.py
@@ -705,7 +705,11 @@ def common_chat_logic(
except Exception as e:
logger.warning("Failed to emit inference event: %s", e, exc_info=True)
- scope._span.set_status(Status(StatusCode.OK))
+ # OK is sticky in the OTel SDK: a later set_status(ERROR) is ignored.
+ # Stream failures set scope._stream_error before this runs, then
+ # handle_exception records ERROR once these attributes are written.
+ if getattr(scope, "_stream_error", None) is None:
+ scope._span.set_status(Status(StatusCode.OK))
# Span status and metrics
if not disable_metrics:
diff --git a/sdk/python/tests/test_litellm_stream_span_lifecycle.py b/sdk/python/tests/test_litellm_stream_span_lifecycle.py
new file mode 100644
index 00000000..4deb3550
--- /dev/null
+++ b/sdk/python/tests/test_litellm_stream_span_lifecycle.py
@@ -0,0 +1,627 @@
+# pylint: disable=missing-function-docstring
+"""Span lifecycle for LiteLLM streaming.
+
+Released ``CustomStreamWrapper`` (through 1.105.1) is a class iterator.
+``async def aclose(self)`` arrived on the 1.84 line. Sync ``close(self) -> None``,
+``__enter__``/``__exit__(self, *exc_info)`` and ``__aenter__``/``__aexit__`` exist
+only on unreleased 1.107, where ``__exit__`` is ``self.close()`` and ``__aexit__``
+is ``await self.aclose()``. A bare ``for`` / ``async for`` calls none of those.
+"""
+
+import asyncio
+import gc
+
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+from opentelemetry.trace import StatusCode
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.litellm.async_litellm import acompletion
+from openlit.instrumentation.litellm.litellm import completion
+from openlit.semcov import SemanticConvention
+
+REQUEST_KWARGS = {
+ "model": "gpt-4o-mini",
+ "stream": True,
+ "messages": [{"role": "user", "content": "hi"}],
+}
+
+
+def _chunk(text, finish_reason=None, usage=None):
+ chunk = {
+ "id": "chatcmpl-1",
+ "model": "gpt-4o-mini",
+ "choices": [{"delta": {"content": text}, "finish_reason": finish_reason}],
+ }
+ if usage is not None:
+ chunk["usage"] = usage
+ return chunk
+
+
+CONTENT = _chunk("Hello")
+STOP = _chunk(
+ " world",
+ finish_reason="stop",
+ usage={"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
+)
+
+
[email protected](autouse=True)
+def _reset_config():
+ OpenlitConfig.reset_to_defaults()
+ yield
+ OpenlitConfig.reset_to_defaults()
+
+
+def _tracer_with_exporter():
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return provider.get_tracer(__name__), exporter
+
+
+class _Iter:
+ """CustomStreamWrapper through 1.105.1: ``__iter__`` / ``__next__`` only."""
+
+ def __init__(self, chunks, raise_after=None):
+ self._chunks = list(chunks)
+ self._raise_after = raise_after
+ self._index = 0
+ self.sync_closed = False
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ if self._raise_after is not None and self._index >= self._raise_after:
+ raise RuntimeError("stream failed")
+ if self._index >= len(self._chunks):
+ raise StopIteration
+ chunk = self._chunks[self._index]
+ self._index += 1
+ return chunk
+
+
+class _ClosingIter(_Iter):
+ """1.107 sync half. ``__exit__(self, *exc_info)`` calls ``close() -> None``."""
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc_info):
+ self.close()
+
+ def close(self):
+ self.sync_closed = True
+
+
+class _SuppressingExit(_ClosingIter):
+ def __exit__(self, *exc_info):
+ self.close()
+ return True
+
+
+class _RaisingExit(_ClosingIter):
+ def __exit__(self, *exc_info):
+ raise RuntimeError("wrapped exit blew up")
+
+
+class _AsyncIter:
+ """1.84 through 1.105.1: ``__anext__`` and ``async def aclose(self)``."""
+
+ def __init__(self, chunks, raise_after=None):
+ self._chunks = list(chunks)
+ self._raise_after = raise_after
+ self._index = 0
+ self.sync_closed = False
+ self.async_closed = False
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ if self._raise_after is not None and self._index >= self._raise_after:
+ raise RuntimeError("stream failed")
+ if self._index >= len(self._chunks):
+ raise StopAsyncIteration
+ chunk = self._chunks[self._index]
+ self._index += 1
+ return chunk
+
+ async def aclose(self):
+ self.async_closed = True
+
+
+class _DevAsyncIter(_AsyncIter):
+ """1.107 async half. ``__aexit__`` awaits ``aclose``; ``close`` stays sync."""
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, *exc_info):
+ await self.aclose()
+
+ def close(self):
+ self.sync_closed = True
+
+
+class _SuppressingAexit(_DevAsyncIter):
+ async def __aexit__(self, *exc_info):
+ await self.aclose()
+ return True
+
+
+class _RaisingAexit(_DevAsyncIter):
+ async def __aexit__(self, *exc_info):
+ raise RuntimeError("wrapped aexit blew up")
+
+
+class _CancellingAsync(_AsyncIter):
+ async def __anext__(self):
+ if self._index == 0:
+ self._index += 1
+ return self._chunks[0]
+ raise asyncio.CancelledError()
+
+
+def _asource(raw):
+ """acompletion's wrapper awaits the wrapped callable."""
+
+ async def _create(*_args, **_kwargs):
+ return raw
+
+ return _create
+
+
+def _factory(tracer, *, is_async):
+ maker = acompletion if is_async else completion
+ return maker(
+ version="test",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=True,
+ metrics=None,
+ disable_metrics=True,
+ event_provider=None,
+ )
+
+
+def _finish_reasons(span):
+ return tuple(span.attributes[SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON])
+
+
+def _assert_partial(span):
+ assert span.status.status_code == StatusCode.OK
+ assert _finish_reasons(span) == ("incomplete",)
+ assert "Hello" in span.attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
+ assert "world" not in span.attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
+ assert span.attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 0
+ assert SemanticConvention.ERROR_TYPE not in span.attributes
+
+
+def _assert_complete(span):
+ assert span.status.status_code == StatusCode.OK
+ assert _finish_reasons(span) == ("stop",)
+ output = span.attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
+ assert "Hello" in output and "world" in output
+ assert span.attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 5
+ assert span.attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 3
+
+
+def _assert_provider_error(span):
+ assert span.status.status_code == StatusCode.ERROR
+ assert span.attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"
+ assert _finish_reasons(span) == ("error",)
+ assert "Hello" in span.attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
+ assert any(event.name == "exception" for event in span.events)
+
+
+def test_sync_bare_for_break_exports_when_released():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+ raw = _Iter([CONTENT, STOP])
+
+ def run():
+ stream = wrapper(lambda *a, **k: raw, None, (), REQUEST_KWARGS)
+ for _ in stream:
+ break
+
+ run()
+ gc.collect()
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+ assert raw.sync_closed is False
+
+
+def test_sync_held_stream_stays_open_until_released():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+ chunks = [CONTENT, _chunk(" there"), STOP]
+ stream = wrapper(lambda *a, **k: _Iter(chunks), None, (), REQUEST_KWARGS)
+
+ iterator = iter(stream)
+ assert next(iterator)["choices"][0]["delta"]["content"] == "Hello"
+ assert not exporter.get_finished_spans()
+ assert next(iterator)["choices"][0]["delta"]["content"] == " there"
+
+ del stream, iterator
+ gc.collect()
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ output = spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
+ assert "Hello" in output and "there" in output
+ assert "world" not in output
+ assert _finish_reasons(spans[0]) == ("incomplete",)
+
+
+def test_sync_with_break_on_released_iterator():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: _Iter([CONTENT, STOP]), None, (), REQUEST_KWARGS)
+ with stream as traced:
+ for _ in traced:
+ break
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+def test_sync_with_break_calls_litellm_close():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+ raw = _ClosingIter([CONTENT, STOP])
+
+ stream = wrapper(lambda *a, **k: raw, None, (), REQUEST_KWARGS)
+ with stream as traced:
+ for _ in traced:
+ break
+
+ assert raw.sync_closed is True
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+def test_sync_close_returns_none_and_finalizes():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+ raw = _ClosingIter([CONTENT, STOP])
+
+ stream = wrapper(lambda *a, **k: raw, None, (), REQUEST_KWARGS)
+ next(stream)
+ assert stream.close() is None
+ assert raw.sync_closed is True
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+def test_sync_close_without_wrapped_close_still_finalizes():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: _Iter([CONTENT, STOP]), None, (), REQUEST_KWARGS)
+ next(stream)
+ assert stream.close() is None
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+def test_sync_full_consumption_exports_stop_once():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+ raw = _ClosingIter([CONTENT, STOP])
+
+ stream = wrapper(lambda *a, **k: raw, None, (), REQUEST_KWARGS)
+ with stream as traced:
+ for _ in traced:
+ pass
+ traced.close()
+
+ assert raw.sync_closed is True
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_complete(spans[0])
+
+
+def test_sync_provider_error_marks_span_error():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(
+ lambda *a, **k: _Iter([CONTENT, STOP], raise_after=1),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ with pytest.raises(RuntimeError, match="stream failed"):
+ with stream as traced:
+ for _ in traced:
+ pass
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_provider_error(spans[0])
+
+
+def test_sync_caller_exception_stays_ok():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(lambda *a, **k: _Iter([CONTENT, STOP]), None, (), REQUEST_KWARGS)
+ with pytest.raises(ValueError, match="caller"):
+ with stream as traced:
+ for _ in traced:
+ raise ValueError("caller")
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+def test_sync_suppressing_exit_hides_caller_exception():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(
+ lambda *a, **k: _SuppressingExit([CONTENT, STOP]), None, (), REQUEST_KWARGS
+ )
+ with stream as traced:
+ for _ in traced:
+ raise ValueError("caller")
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+def test_sync_exception_from_wrapped_exit_is_error():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=False)
+
+ stream = wrapper(
+ lambda *a, **k: _RaisingExit([CONTENT, STOP]), None, (), REQUEST_KWARGS
+ )
+ with pytest.raises(RuntimeError, match="wrapped exit blew up"):
+ with stream as traced:
+ for _ in traced:
+ break
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_provider_error(spans[0])
+
+
+async def test_async_bare_for_break_exports_when_released():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+ raw = _AsyncIter([CONTENT, STOP])
+
+ async def run():
+ stream = await wrapper(_asource(raw), None, (), REQUEST_KWARGS)
+ async for _ in stream:
+ break
+
+ await run()
+ gc.collect()
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+ assert raw.async_closed is False
+
+
+async def test_async_held_stream_stays_open_until_released():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+ chunks = [CONTENT, _chunk(" there"), STOP]
+ stream = await wrapper(_asource(_AsyncIter(chunks)), None, (), REQUEST_KWARGS)
+
+ assert (await anext(stream))["choices"][0]["delta"]["content"] == "Hello"
+ assert not exporter.get_finished_spans()
+ assert (await anext(stream))["choices"][0]["delta"]["content"] == " there"
+
+ del stream
+ gc.collect()
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ output = spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
+ assert "Hello" in output and "there" in output
+ assert "world" not in output
+
+
+async def test_async_with_on_aclose_only_stream_calls_aclose():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+ raw = _AsyncIter([CONTENT, STOP])
+
+ stream = await wrapper(_asource(raw), None, (), REQUEST_KWARGS)
+ async with stream as traced:
+ async for _ in traced:
+ break
+
+ assert raw.async_closed is True
+ assert raw.sync_closed is False
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+async def test_async_with_on_107_calls_aclose_not_sync_close():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+ raw = _DevAsyncIter([CONTENT, STOP])
+
+ stream = await wrapper(_asource(raw), None, (), REQUEST_KWARGS)
+ async with stream as traced:
+ async for _ in traced:
+ break
+
+ assert raw.async_closed is True
+ assert raw.sync_closed is False
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+async def test_async_aclose_finalizes_without_context_manager():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+ raw = _AsyncIter([CONTENT, STOP])
+
+ stream = await wrapper(_asource(raw), None, (), REQUEST_KWARGS)
+ await anext(stream)
+ await stream.aclose()
+
+ assert raw.async_closed is True
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+async def test_async_sync_close_is_not_a_coroutine():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+ raw = _DevAsyncIter([CONTENT, STOP])
+
+ stream = await wrapper(_asource(raw), None, (), REQUEST_KWARGS)
+ await anext(stream)
+ result = stream.close()
+
+ assert result is None
+ assert not asyncio.iscoroutine(result)
+ assert raw.sync_closed is True
+ assert raw.async_closed is False
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+async def test_async_full_consumption_exports_stop_once():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+ raw = _DevAsyncIter([CONTENT, STOP])
+
+ stream = await wrapper(_asource(raw), None, (), REQUEST_KWARGS)
+ async with stream as traced:
+ async for _ in traced:
+ pass
+ await traced.aclose()
+
+ assert raw.async_closed is True
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_complete(spans[0])
+
+
+async def test_async_provider_error_marks_span_error():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(
+ _asource(_AsyncIter([CONTENT, STOP], raise_after=1)),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ with pytest.raises(RuntimeError, match="stream failed"):
+ async with stream as traced:
+ async for _ in traced:
+ pass
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_provider_error(spans[0])
+
+
+async def test_async_caller_exception_stays_ok():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(
+ _asource(_AsyncIter([CONTENT, STOP])), None, (), REQUEST_KWARGS
+ )
+ with pytest.raises(ValueError, match="caller"):
+ async with stream as traced:
+ async for _ in traced:
+ raise ValueError("caller")
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+async def test_async_suppressing_aexit_hides_caller_exception():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(
+ _asource(_SuppressingAexit([CONTENT, STOP])), None, (), REQUEST_KWARGS
+ )
+ async with stream as traced:
+ async for _ in traced:
+ raise ValueError("caller")
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])
+
+
+async def test_async_exception_from_wrapped_aexit_is_error():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(
+ _asource(_RaisingAexit([CONTENT, STOP])), None, (), REQUEST_KWARGS
+ )
+ with pytest.raises(RuntimeError, match="wrapped aexit blew up"):
+ async with stream as traced:
+ async for _ in traced:
+ break
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_provider_error(spans[0])
+
+
+async def test_async_cancellation_finalizes_incomplete():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer, is_async=True)
+
+ stream = await wrapper(
+ _asource(_CancellingAsync([CONTENT, STOP])), None, (), REQUEST_KWARGS
+ )
+ await anext(stream)
+ with pytest.raises(asyncio.CancelledError):
+ await anext(stream)
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ _assert_partial(spans[0])