Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(premai): make the streaming wrapper iterable and end its span once (#1610)

BASIL K AJIddb19cd8e2ee
diff --git a/sdk/python/src/openlit/instrumentation/premai/premai.py b/sdk/python/src/openlit/instrumentation/premai/premai.py
index 0b4ed49e..a0ca2391 100644
--- a/sdk/python/src/openlit/instrumentation/premai/premai.py
+++ b/sdk/python/src/openlit/instrumentation/premai/premai.py
@@ -71,26 +71,104 @@ def chat(
self._tbt = 0
self._server_address = server_address
self._server_port = server_port
+ self._streaming_response_processed = False
+ self._stream_error = None
+ # premai 0.3 ChatCompletionResponseStreamContainer defines __iter__
+ # only and returns a new generator each call. Cache the first one.
+ self._iterator = None
def __enter__(self):
+ # The 0.3 container is not a context manager. Callers that use
+ # ``with`` are using this wrapper.
+ enter = getattr(self.__wrapped__, "__enter__", None)
+ if enter is not None:
+ enter()
return self
def __exit__(self, exc_type, exc_value, traceback):
- pass
+ # Return what the wrapped stream returns: a context manager that
+ # suppresses an exception says so with a truthy __exit__, and
+ # swallowing that value would force the exception to re-raise.
+ # The 0.3 container has no __exit__; skip the forward in that case.
+ suppress = False
+ exit_fn = getattr(self.__wrapped__, "__exit__", None)
+ try:
+ if exit_fn is not None:
+ suppress = exit_fn(exc_type, exc_value, traceback)
+ finally:
+ # A raise from user code is not a Prem failure. The generator
+ # already stored its own error on _stream_error in __next__.
+ self.close()
+ return suppress
def __getattr__(self, name):
"""Delegate attribute access to the wrapped object."""
return getattr(self.__wrapped__, name)
def __iter__(self):
+ return self
+
+ def _ensure_iterator(self):
+ """Open the wrapped iterable once.
+
+ ``iter()`` covers both a real iterator and the 0.3 container, whose
+ ``__iter__`` returns a generator and which has no ``__next__``.
+ A second ``iter()`` on that container opens another generator over
+ the same httpx body.
+ """
+ if self._iterator is None:
+ self._iterator = iter(self.__wrapped__)
+ return self._iterator
+
+ def __next__(self):
try:
- chunk = self.__wrapped__.__next__()
+ chunk = next(self._ensure_iterator())
process_chunk(self, chunk)
return chunk
+ except StopIteration:
+ self._finalize_streaming_span(ended_cleanly=True)
+ raise
+ except Exception as exc:
+ self._stream_error = exc
+ raise
- finally:
- try:
- with self._span:
+ def _close_iterator(self):
+ """Close the generator this wrapper opened.
+
+ The 0.3 container has no ``close``. Closing the generator raises
+ ``GeneratorExit`` inside its ``with self._stream`` block, which is
+ what releases the httpx response. ``_iterator`` stays None until
+ the first read, so ``close()`` before iteration does not enter
+ that body.
+ """
+ iterator = self._iterator
+ closer = getattr(iterator, "close", None) if iterator is not None else None
+ if closer is None:
+ return
+ try:
+ closer()
+ except Exception:
+ pass
+
+ def _finalize_streaming_span(self, ended_cleanly):
+ """End the span once.
+
+ ``ended_cleanly`` is True only for StopIteration. An empty finish
+ reason is otherwise a cancel. A failure raised by the stream is
+ recorded after attributes are set, so partial text stays on the
+ span and the status is ERROR rather than the OK that
+ ``process_streaming_chat_response`` writes when nothing failed.
+ """
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ if self._stream_error is not None and not self._finish_reason:
+ self._finish_reason = "error"
+ elif not ended_cleanly and not self._finish_reason:
+ self._finish_reason = "incomplete"
+ try:
+ with self._span:
+ try:
process_streaming_chat_response(
self,
pricing_info=pricing_info,
@@ -102,9 +180,29 @@ def chat(
version=version,
event_provider=event_provider,
)
+ finally:
+ if self._stream_error is not None:
+ handle_exception(self._span, self._stream_error)
+ except Exception as e:
+ handle_exception(self._span, e)
- except Exception as e:
- handle_exception(self._span, e)
+ def close(self):
+ """Close the cached generator and finalize the span if not ended."""
+ try:
+ self._close_iterator()
+ finally:
+ self._finalize_streaming_span(ended_cleanly=False)
+
+ def __del__(self):
+ """Finalize a span abandoned by a bare ``for`` loop.
+
+ The wrapper is the iterator, so ``break`` never calls ``close()``.
+ Prem's documented call is that bare loop, not a context manager.
+ """
+ try:
+ self.close()
+ except Exception:
+ pass
def wrapper(wrapped, instance, args, kwargs):
"""
diff --git a/sdk/python/src/openlit/instrumentation/premai/utils.py b/sdk/python/src/openlit/instrumentation/premai/utils.py
index 9214d633..4cc1f827 100644
--- a/sdk/python/src/openlit/instrumentation/premai/utils.py
+++ b/sdk/python/src/openlit/instrumentation/premai/utils.py
@@ -279,6 +279,29 @@ def emit_inference_event(
logger.warning("Failed to emit inference event: %s", e, exc_info=True)
+def _chunk_mapping(chunk):
+ """Return a dict for one premai stream chunk.
+
+ 0.3 streams yield attrs objects. ``response_as_dict`` returns those
+ unchanged, and this reader only understands dicts. ``to_dict`` on the
+ chunk already recurses into choices and usage. Dict chunks pass through.
+ This stays in the premai package: ``response_as_dict`` is shared by every
+ provider.
+ """
+
+ parsed = response_as_dict(chunk)
+ if isinstance(parsed, dict):
+ return parsed
+ to_dict = getattr(chunk, "to_dict", None)
+ if not callable(to_dict):
+ return {}
+ try:
+ parsed = to_dict()
+ except Exception:
+ return {}
+ return parsed if isinstance(parsed, dict) else {}
+
+
def process_chunk(scope, chunk):
"""
Process a chunk of response data and update state.
@@ -290,7 +313,7 @@ def process_chunk(scope, chunk):
if len(scope._timestamps) == 1:
scope._ttft = calculate_ttft(scope._timestamps, scope._start_time)
- chunked = response_as_dict(chunk)
+ chunked = _chunk_mapping(chunk)
choices = chunked.get("choices", []) if isinstance(chunked, dict) else []
if choices:
first_choice = choices[0] if isinstance(choices[0], dict) else {}
@@ -561,8 +584,10 @@ def common_chat_logic(
except Exception as e:
logger.warning("Failed to emit inference event: %s", e, exc_info=True)
- # Span status and metrics
- scope._span.set_status(Status(StatusCode.OK))
+ # Span status and metrics. OTel drops set_status(ERROR) once status is
+ # OK, and a stream failure records ERROR after this returns.
+ if getattr(scope, "_stream_error", None) is None:
+ scope._span.set_status(Status(StatusCode.OK))
if not disable_metrics:
inter_chunk_durations = None
if getattr(scope, "_timestamps", None) and len(scope._timestamps) > 1:
diff --git a/sdk/python/tests/test_premai_stream_span_lifecycle.py b/sdk/python/tests/test_premai_stream_span_lifecycle.py
new file mode 100644
index 00000000..a2403786
--- /dev/null
+++ b/sdk/python/tests/test_premai_stream_span_lifecycle.py
@@ -0,0 +1,455 @@
+# pylint: disable=protected-access, duplicate-code, missing-function-docstring, too-few-public-methods
+"""Regression tests: the premai streaming wrapper must be iterable, and must
+end its span on every exit path.
+
+`TracedSyncStream` defined `__iter__` with a `__next__` body: it pulled one
+chunk, ran `process_chunk`, and returned that chunk in place of an iterator.
+`for chunk in stream:` therefore raised `TypeError: iter() returned non-iterator`
+and installing openlit turned a working premai stream into a crash. The
+`finally` around it was the second half — it finalized the span after the first
+chunk, so the one span that did get exported carried the first delta only and
+left `gen_ai.response.id` empty, because the id is read in the finish-reason
+branch an aborted iteration never reaches.
+
+premai 0.3 `ChatCompletionResponseStreamContainer` defines `__iter__` only and
+returns a generator. It has no `__next__`, `close`, `__enter__`, or `__exit__`.
+Chunks are attrs objects with `to_dict`, not dicts. `FakeSyncStream` below is
+the iterator from the original repro. `IterOnlyContainer` is the 0.3 container.
+"""
+
+import gc
+import time
+
+import pytest
+from opentelemetry import trace as trace_api
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+from opentelemetry.trace import StatusCode
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.premai import premai as premai_mod
+from openlit.semcov import SemanticConvention
+
+REQUEST_KWARGS = {
+ "model": "gpt-4o-mini",
+ "stream": True,
+ "messages": [{"role": "user", "content": "Monitor LLM Applications"}],
+}
+
+# Two premai-shaped chunks: a content delta, then a final chunk with usage.
+CHUNKS = [
+ {
+ "id": "chatcmpl-1",
+ "model": "gpt-4o-mini",
+ "choices": [{"delta": {"content": "partial"}, "finish_reason": None}],
+ },
+ {
+ "id": "chatcmpl-1",
+ "model": "gpt-4o-mini",
+ "choices": [{"delta": {"content": " answer"}, "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 5},
+ },
+]
+
+
+def _tracer_with_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return provider.get_tracer(__name__), exporter
+
+
+def _factory(tracer):
+ return premai_mod.chat(
+ version="test",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=True,
+ metrics=None,
+ disable_metrics=True,
+ event_provider=None,
+ )
+
+
+class FakeSyncStream:
+ """Iterator plus context manager, the shape the original repro used.
+
+ premai 0.3 does not return this. `IterOnlyContainer` is that object.
+ """
+
+ def __init__(self):
+ self._it = iter(CHUNKS)
+ self.closed = False
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ return next(self._it)
+
+ def close(self):
+ self.closed = True
+
+
+class IterOnlyContainer:
+ """premai 0.3 `ChatCompletionResponseStreamContainer`.
+
+ `__iter__` returns a generator. There is no `__next__`, `close`,
+ `__enter__`, or `__exit__`. The generator's `finally` stands in for the
+ container's `with self._stream` block.
+ """
+
+ def __init__(self, chunks, box, raise_after=None):
+ self._chunks = list(chunks)
+ self._box = box
+ self._raise_after = raise_after
+
+ def __iter__(self):
+ chunks = self._chunks
+ box = self._box
+ raise_after = self._raise_after
+
+ def generator():
+ try:
+ for index, chunk in enumerate(chunks):
+ if raise_after is not None and index >= raise_after:
+ raise RuntimeError("stream failed")
+ yield chunk
+ finally:
+ box["closed"] = True
+
+ return generator()
+
+
+class AttrsChunk:
+ """Stand-in for a premai attrs chunk. `to_dict` returns the wire dict."""
+
+ def __init__(self, payload):
+ self._payload = payload
+
+ def to_dict(self):
+ """Return the chunk the streaming reader expects."""
+ return self._payload
+
+
+def _one_finished_span(exporter):
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1, f"expected exactly one span, got {len(spans)}"
+ return spans[0]
+
+
+def test_the_stream_is_iterable():
+ """`for chunk in stream:` must work. It raised TypeError before the fix."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ received = list(stream)
+
+ assert received == CHUNKS, "every chunk must reach the caller unchanged"
+ time.sleep(0.05)
+ _one_finished_span(exporter)
+
+
+def test_full_consumption_exports_exactly_one_span_with_usage():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ for _ in stream:
+ pass
+
+ time.sleep(0.05)
+ span = _one_finished_span(exporter)
+ attrs = span.attributes
+ # The id, model and finish reason are read in the finish-reason branch,
+ # which only a completed iteration reaches. Ending after the first chunk
+ # left all three unset however many chunks the stream really had.
+ assert attrs.get(SemanticConvention.GEN_AI_RESPONSE_ID) == "chatcmpl-1"
+ assert list(attrs.get(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON)) == ["stop"]
+ # Both deltas must be in the recorded answer, not just the first.
+ assert "partial answer" in str(attrs)
+ # premai estimates stream token counts from the text rather than reading
+ # `usage`, so assert they are populated rather than pinning the estimate.
+ assert attrs.get(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS) > 0
+ assert attrs.get(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS) > 0
+
+
+def test_break_before_exhaustion_still_exports_the_span():
+ """A caller that leaves the block early must not leak the span."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ for _ in s:
+ break
+
+ time.sleep(0.05)
+ _one_finished_span(exporter)
+
+
+def test_close_finalizes_the_span_once():
+ """close() must finalize, and exhaustion afterwards must not double-end."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ next(iter(stream))
+ stream.close()
+ stream.close() # the guard makes the second call a no-op
+
+ time.sleep(0.05)
+ _one_finished_span(exporter)
+
+
+def test_context_manager_exit_finalizes_without_iterating_to_the_end():
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ with stream as s:
+ next(iter(s))
+
+ time.sleep(0.05)
+ _one_finished_span(exporter)
+
+
+def test_exception_suppression_by_the_wrapped_stream_is_honoured():
+ """A wrapped stream that suppresses an exception must still suppress it.
+
+ `__exit__` forwards to the wrapped stream; returning None instead of what
+ it returned would turn a suppressed exception back into a raised one.
+ """
+
+ class SuppressingStream(FakeSyncStream):
+ """A stream whose __exit__ claims it handled the exception."""
+
+ def __exit__(self, *exc):
+ return True # "I handled it"
+
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+
+ stream = wrapper(lambda *a, **k: SuppressingStream(), None, (), REQUEST_KWARGS)
+ with stream:
+ raise ValueError("the wrapped stream says it handles this")
+
+ time.sleep(0.05)
+ _one_finished_span(exporter)
+
+
+def test_the_span_is_detached_from_context():
+ """The wrapper attaches a context token; nothing may leak into the caller."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+
+ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)
+ for _ in stream:
+ pass
+
+ time.sleep(0.05)
+ _one_finished_span(exporter)
+ assert trace_api.get_current_span().get_span_context().span_id == 0
+
+
+def _finish_reasons(span):
+ return list(span.attributes.get(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON))
+
+
+def test_iter_only_container_yields_every_chunk_and_one_span():
+ """The 0.3 container has no __next__. list() must still return both chunks."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+ box = {"closed": False}
+
+ stream = wrapper(
+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ received = list(stream)
+
+ assert received == CHUNKS
+ assert box["closed"]
+ time.sleep(0.05)
+ span = _one_finished_span(exporter)
+ assert span.attributes.get(SemanticConvention.GEN_AI_RESPONSE_ID) == "chatcmpl-1"
+ assert _finish_reasons(span) == ["stop"]
+ assert "partial answer" in str(span.attributes)
+
+
+def test_iter_only_container_used_as_context_manager_does_not_require_enter():
+ """``with stream`` is the wrapper's protocol. The container has no __enter__."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+ box = {"closed": False}
+
+ stream = wrapper(
+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ with stream as traced:
+ assert next(traced) == CHUNKS[0]
+ assert box["closed"]
+ time.sleep(0.05)
+ span = _one_finished_span(exporter)
+ assert _finish_reasons(span) == ["incomplete"]
+
+
+def test_close_on_iter_only_container_closes_the_generator_once():
+ """close() closes the generator the container returned, not the container."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+ box = {"closed": False}
+
+ stream = wrapper(
+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ next(stream)
+ stream.close()
+ stream.close()
+
+ assert box["closed"]
+ time.sleep(0.05)
+ span = _one_finished_span(exporter)
+ assert _finish_reasons(span) == ["incomplete"]
+ assert "partial" in str(span.attributes)
+
+
+def test_bare_break_exports_the_span_when_the_wrapper_is_released():
+ """Prem's documented call is a bare for-loop. break must not leak the span."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+ box = {"closed": False}
+
+ def run():
+ traced = wrapper(
+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ for _ in traced:
+ break
+
+ run()
+ gc.collect()
+
+ assert box["closed"]
+ span = _one_finished_span(exporter)
+ assert span.status.status_code == StatusCode.OK
+ assert _finish_reasons(span) == ["incomplete"]
+ assert "partial" in str(span.attributes)
+
+
+def test_held_iter_only_stream_stays_open_until_released():
+ """A kept reference must stay resumable. Finalizing on break would drop the tail."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+ box = {"closed": False}
+
+ stream = wrapper(
+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ first = next(stream)
+
+ assert first == CHUNKS[0]
+ assert box["closed"] is False
+ assert len(exporter.get_finished_spans()) == 0
+ assert next(stream) == CHUNKS[1]
+
+ del stream
+ gc.collect()
+
+ assert box["closed"]
+ _one_finished_span(exporter)
+
+
+def test_stream_error_is_recorded_after_partial_text():
+ """A failure from the generator stays on the span with the text already seen."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+ box = {"closed": False}
+
+ def run():
+ traced = wrapper(
+ lambda *a, **k: IterOnlyContainer(CHUNKS, box, raise_after=1),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ with pytest.raises(RuntimeError, match="stream failed"):
+ list(traced)
+
+ run()
+ gc.collect()
+
+ assert box["closed"]
+ span = _one_finished_span(exporter)
+ assert span.status.status_code == StatusCode.ERROR
+ assert _finish_reasons(span) == ["error"]
+ assert "partial" in str(span.attributes)
+
+
+def test_attrs_chunks_contribute_content_and_response_id():
+ """0.3 yields attrs objects. to_dict() is what puts the text on the span."""
+ tracer, exporter = _tracer_with_exporter()
+ wrapper = _factory(tracer)
+ chunks = [AttrsChunk(chunk) for chunk in CHUNKS]
+ box = {"closed": False}
+
+ stream = wrapper(
+ lambda *a, **k: IterOnlyContainer(chunks, box),
+ None,
+ (),
+ REQUEST_KWARGS,
+ )
+ received = list(stream)
+
+ assert received == chunks
+ time.sleep(0.05)
+ span = _one_finished_span(exporter)
+ assert span.attributes.get(SemanticConvention.GEN_AI_RESPONSE_ID) == "chatcmpl-1"
+ assert _finish_reasons(span) == ["stop"]
+ assert "partial answer" in str(span.attributes)
+ assert span.attributes.get(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS) > 0
+
+
+def test_real_0_3_container_exposes_iter_only():
+ """Lock the surface these tests stand in for, when that SDK is installed."""
+ pytest.importorskip("premai")
+ try:
+ from premai.api.chat_completions.v1_chat_completions_create import (
+ ChatCompletionResponseStreamContainer as container,
+ )
+ except ImportError:
+ pytest.skip("premai.api is the 0.3 client; this install is newer")
+
+ present = [
+ name
+ for name in ("__iter__", "__next__", "close", "__enter__", "__exit__")
+ if name in vars(container)
+ ]
+ assert present == ["__iter__"]