Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(premai): make the streaming wrapper iterable and end its span once (#1610)
BASIL K AJIddb19cd8e2ee
diff --git a/sdk/python/src/openlit/instrumentation/premai/premai.py b/sdk/python/src/openlit/instrumentation/premai/premai.pyindex 0b4ed49e..a0ca2391 100644--- a/sdk/python/src/openlit/instrumentation/premai/premai.py+++ b/sdk/python/src/openlit/instrumentation/premai/premai.py@@ -71,26 +71,104 @@ def chat(self._tbt = 0self._server_address = server_addressself._server_port = server_port+ self._streaming_response_processed = False+ self._stream_error = None+ # premai 0.3 ChatCompletionResponseStreamContainer defines __iter__+ # only and returns a new generator each call. Cache the first one.+ self._iterator = Nonedef __enter__(self):+ # The 0.3 container is not a context manager. Callers that use+ # ``with`` are using this wrapper.+ enter = getattr(self.__wrapped__, "__enter__", None)+ if enter is not None:+ enter()return selfdef __exit__(self, exc_type, exc_value, traceback):- pass+ # Return what the wrapped stream returns: a context manager that+ # suppresses an exception says so with a truthy __exit__, and+ # swallowing that value would force the exception to re-raise.+ # The 0.3 container has no __exit__; skip the forward in that case.+ suppress = False+ exit_fn = getattr(self.__wrapped__, "__exit__", None)+ try:+ if exit_fn is not None:+ suppress = exit_fn(exc_type, exc_value, traceback)+ finally:+ # A raise from user code is not a Prem failure. The generator+ # already stored its own error on _stream_error in __next__.+ self.close()+ return suppressdef __getattr__(self, name):"""Delegate attribute access to the wrapped object."""return getattr(self.__wrapped__, name)def __iter__(self):+ return self++ def _ensure_iterator(self):+ """Open the wrapped iterable once.++ ``iter()`` covers both a real iterator and the 0.3 container, whose+ ``__iter__`` returns a generator and which has no ``__next__``.+ A second ``iter()`` on that container opens another generator over+ the same httpx body.+ """+ if self._iterator is None:+ self._iterator = iter(self.__wrapped__)+ return self._iterator++ def __next__(self):try:- chunk = self.__wrapped__.__next__()+ chunk = next(self._ensure_iterator())process_chunk(self, chunk)return chunk+ except StopIteration:+ self._finalize_streaming_span(ended_cleanly=True)+ raise+ except Exception as exc:+ self._stream_error = exc+ raise- finally:- try:- with self._span:+ def _close_iterator(self):+ """Close the generator this wrapper opened.++ The 0.3 container has no ``close``. Closing the generator raises+ ``GeneratorExit`` inside its ``with self._stream`` block, which is+ what releases the httpx response. ``_iterator`` stays None until+ the first read, so ``close()`` before iteration does not enter+ that body.+ """+ iterator = self._iterator+ closer = getattr(iterator, "close", None) if iterator is not None else None+ if closer is None:+ return+ try:+ closer()+ except Exception:+ pass++ def _finalize_streaming_span(self, ended_cleanly):+ """End the span once.++ ``ended_cleanly`` is True only for StopIteration. An empty finish+ reason is otherwise a cancel. A failure raised by the stream is+ recorded after attributes are set, so partial text stays on the+ span and the status is ERROR rather than the OK that+ ``process_streaming_chat_response`` writes when nothing failed.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ if self._stream_error is not None and not self._finish_reason:+ self._finish_reason = "error"+ elif not ended_cleanly and not self._finish_reason:+ self._finish_reason = "incomplete"+ try:+ with self._span:+ try:process_streaming_chat_response(self,pricing_info=pricing_info,@@ -102,9 +180,29 @@ def chat(version=version,event_provider=event_provider,)+ finally:+ if self._stream_error is not None:+ handle_exception(self._span, self._stream_error)+ except Exception as e:+ handle_exception(self._span, e)- except Exception as e:- handle_exception(self._span, e)+ def close(self):+ """Close the cached generator and finalize the span if not ended."""+ try:+ self._close_iterator()+ finally:+ self._finalize_streaming_span(ended_cleanly=False)++ def __del__(self):+ """Finalize a span abandoned by a bare ``for`` loop.++ The wrapper is the iterator, so ``break`` never calls ``close()``.+ Prem's documented call is that bare loop, not a context manager.+ """+ try:+ self.close()+ except Exception:+ passdef wrapper(wrapped, instance, args, kwargs):"""diff --git a/sdk/python/src/openlit/instrumentation/premai/utils.py b/sdk/python/src/openlit/instrumentation/premai/utils.pyindex 9214d633..4cc1f827 100644--- a/sdk/python/src/openlit/instrumentation/premai/utils.py+++ b/sdk/python/src/openlit/instrumentation/premai/utils.py@@ -279,6 +279,29 @@ def emit_inference_event(logger.warning("Failed to emit inference event: %s", e, exc_info=True)+def _chunk_mapping(chunk):+ """Return a dict for one premai stream chunk.++ 0.3 streams yield attrs objects. ``response_as_dict`` returns those+ unchanged, and this reader only understands dicts. ``to_dict`` on the+ chunk already recurses into choices and usage. Dict chunks pass through.+ This stays in the premai package: ``response_as_dict`` is shared by every+ provider.+ """++ parsed = response_as_dict(chunk)+ if isinstance(parsed, dict):+ return parsed+ to_dict = getattr(chunk, "to_dict", None)+ if not callable(to_dict):+ return {}+ try:+ parsed = to_dict()+ except Exception:+ return {}+ return parsed if isinstance(parsed, dict) else {}++def process_chunk(scope, chunk):"""Process a chunk of response data and update state.@@ -290,7 +313,7 @@ def process_chunk(scope, chunk):if len(scope._timestamps) == 1:scope._ttft = calculate_ttft(scope._timestamps, scope._start_time)- chunked = response_as_dict(chunk)+ chunked = _chunk_mapping(chunk)choices = chunked.get("choices", []) if isinstance(chunked, dict) else []if choices:first_choice = choices[0] if isinstance(choices[0], dict) else {}@@ -561,8 +584,10 @@ def common_chat_logic(except Exception as e:logger.warning("Failed to emit inference event: %s", e, exc_info=True)- # Span status and metrics- scope._span.set_status(Status(StatusCode.OK))+ # Span status and metrics. OTel drops set_status(ERROR) once status is+ # OK, and a stream failure records ERROR after this returns.+ if getattr(scope, "_stream_error", None) is None:+ scope._span.set_status(Status(StatusCode.OK))if not disable_metrics:inter_chunk_durations = Noneif getattr(scope, "_timestamps", None) and len(scope._timestamps) > 1:diff --git a/sdk/python/tests/test_premai_stream_span_lifecycle.py b/sdk/python/tests/test_premai_stream_span_lifecycle.pynew file mode 100644index 00000000..a2403786--- /dev/null+++ b/sdk/python/tests/test_premai_stream_span_lifecycle.py@@ -0,0 +1,455 @@+# pylint: disable=protected-access, duplicate-code, missing-function-docstring, too-few-public-methods+"""Regression tests: the premai streaming wrapper must be iterable, and must+end its span on every exit path.++`TracedSyncStream` defined `__iter__` with a `__next__` body: it pulled one+chunk, ran `process_chunk`, and returned that chunk in place of an iterator.+`for chunk in stream:` therefore raised `TypeError: iter() returned non-iterator`+and installing openlit turned a working premai stream into a crash. The+`finally` around it was the second half — it finalized the span after the first+chunk, so the one span that did get exported carried the first delta only and+left `gen_ai.response.id` empty, because the id is read in the finish-reason+branch an aborted iteration never reaches.++premai 0.3 `ChatCompletionResponseStreamContainer` defines `__iter__` only and+returns a generator. It has no `__next__`, `close`, `__enter__`, or `__exit__`.+Chunks are attrs objects with `to_dict`, not dicts. `FakeSyncStream` below is+the iterator from the original repro. `IterOnlyContainer` is the 0.3 container.+"""++import gc+import time++import pytest+from opentelemetry import trace as trace_api+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)+from opentelemetry.trace import StatusCode++from openlit._config import OpenlitConfig+from openlit.instrumentation.premai import premai as premai_mod+from openlit.semcov import SemanticConvention++REQUEST_KWARGS = {+ "model": "gpt-4o-mini",+ "stream": True,+ "messages": [{"role": "user", "content": "Monitor LLM Applications"}],+}++# Two premai-shaped chunks: a content delta, then a final chunk with usage.+CHUNKS = [+ {+ "id": "chatcmpl-1",+ "model": "gpt-4o-mini",+ "choices": [{"delta": {"content": "partial"}, "finish_reason": None}],+ },+ {+ "id": "chatcmpl-1",+ "model": "gpt-4o-mini",+ "choices": [{"delta": {"content": " answer"}, "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 10, "completion_tokens": 5},+ },+]+++def _tracer_with_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _factory(tracer):+ return premai_mod.chat(+ version="test",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=True,+ metrics=None,+ disable_metrics=True,+ event_provider=None,+ )+++class FakeSyncStream:+ """Iterator plus context manager, the shape the original repro used.++ premai 0.3 does not return this. `IterOnlyContainer` is that object.+ """++ def __init__(self):+ self._it = iter(CHUNKS)+ self.closed = False++ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False++ def __iter__(self):+ return self++ def __next__(self):+ return next(self._it)++ def close(self):+ self.closed = True+++class IterOnlyContainer:+ """premai 0.3 `ChatCompletionResponseStreamContainer`.++ `__iter__` returns a generator. There is no `__next__`, `close`,+ `__enter__`, or `__exit__`. The generator's `finally` stands in for the+ container's `with self._stream` block.+ """++ def __init__(self, chunks, box, raise_after=None):+ self._chunks = list(chunks)+ self._box = box+ self._raise_after = raise_after++ def __iter__(self):+ chunks = self._chunks+ box = self._box+ raise_after = self._raise_after++ def generator():+ try:+ for index, chunk in enumerate(chunks):+ if raise_after is not None and index >= raise_after:+ raise RuntimeError("stream failed")+ yield chunk+ finally:+ box["closed"] = True++ return generator()+++class AttrsChunk:+ """Stand-in for a premai attrs chunk. `to_dict` returns the wire dict."""++ def __init__(self, payload):+ self._payload = payload++ def to_dict(self):+ """Return the chunk the streaming reader expects."""+ return self._payload+++def _one_finished_span(exporter):+ spans = exporter.get_finished_spans()+ assert len(spans) == 1, f"expected exactly one span, got {len(spans)}"+ return spans[0]+++def test_the_stream_is_iterable():+ """`for chunk in stream:` must work. It raised TypeError before the fix."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ received = list(stream)++ assert received == CHUNKS, "every chunk must reach the caller unchanged"+ time.sleep(0.05)+ _one_finished_span(exporter)+++def test_full_consumption_exports_exactly_one_span_with_usage():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ for _ in stream:+ pass++ time.sleep(0.05)+ span = _one_finished_span(exporter)+ attrs = span.attributes+ # The id, model and finish reason are read in the finish-reason branch,+ # which only a completed iteration reaches. Ending after the first chunk+ # left all three unset however many chunks the stream really had.+ assert attrs.get(SemanticConvention.GEN_AI_RESPONSE_ID) == "chatcmpl-1"+ assert list(attrs.get(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON)) == ["stop"]+ # Both deltas must be in the recorded answer, not just the first.+ assert "partial answer" in str(attrs)+ # premai estimates stream token counts from the text rather than reading+ # `usage`, so assert they are populated rather than pinning the estimate.+ assert attrs.get(SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS) > 0+ assert attrs.get(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS) > 0+++def test_break_before_exhaustion_still_exports_the_span():+ """A caller that leaves the block early must not leak the span."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ for _ in s:+ break++ time.sleep(0.05)+ _one_finished_span(exporter)+++def test_close_finalizes_the_span_once():+ """close() must finalize, and exhaustion afterwards must not double-end."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ next(iter(stream))+ stream.close()+ stream.close() # the guard makes the second call a no-op++ time.sleep(0.05)+ _one_finished_span(exporter)+++def test_context_manager_exit_finalizes_without_iterating_to_the_end():+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ with stream as s:+ next(iter(s))++ time.sleep(0.05)+ _one_finished_span(exporter)+++def test_exception_suppression_by_the_wrapped_stream_is_honoured():+ """A wrapped stream that suppresses an exception must still suppress it.++ `__exit__` forwards to the wrapped stream; returning None instead of what+ it returned would turn a suppressed exception back into a raised one.+ """++ class SuppressingStream(FakeSyncStream):+ """A stream whose __exit__ claims it handled the exception."""++ def __exit__(self, *exc):+ return True # "I handled it"++ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)++ stream = wrapper(lambda *a, **k: SuppressingStream(), None, (), REQUEST_KWARGS)+ with stream:+ raise ValueError("the wrapped stream says it handles this")++ time.sleep(0.05)+ _one_finished_span(exporter)+++def test_the_span_is_detached_from_context():+ """The wrapper attaches a context token; nothing may leak into the caller."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)++ stream = wrapper(lambda *a, **k: FakeSyncStream(), None, (), REQUEST_KWARGS)+ for _ in stream:+ pass++ time.sleep(0.05)+ _one_finished_span(exporter)+ assert trace_api.get_current_span().get_span_context().span_id == 0+++def _finish_reasons(span):+ return list(span.attributes.get(SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON))+++def test_iter_only_container_yields_every_chunk_and_one_span():+ """The 0.3 container has no __next__. list() must still return both chunks."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)+ box = {"closed": False}++ stream = wrapper(+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),+ None,+ (),+ REQUEST_KWARGS,+ )+ received = list(stream)++ assert received == CHUNKS+ assert box["closed"]+ time.sleep(0.05)+ span = _one_finished_span(exporter)+ assert span.attributes.get(SemanticConvention.GEN_AI_RESPONSE_ID) == "chatcmpl-1"+ assert _finish_reasons(span) == ["stop"]+ assert "partial answer" in str(span.attributes)+++def test_iter_only_container_used_as_context_manager_does_not_require_enter():+ """``with stream`` is the wrapper's protocol. The container has no __enter__."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)+ box = {"closed": False}++ stream = wrapper(+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),+ None,+ (),+ REQUEST_KWARGS,+ )+ with stream as traced:+ assert next(traced) == CHUNKS[0]+ assert box["closed"]+ time.sleep(0.05)+ span = _one_finished_span(exporter)+ assert _finish_reasons(span) == ["incomplete"]+++def test_close_on_iter_only_container_closes_the_generator_once():+ """close() closes the generator the container returned, not the container."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)+ box = {"closed": False}++ stream = wrapper(+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),+ None,+ (),+ REQUEST_KWARGS,+ )+ next(stream)+ stream.close()+ stream.close()++ assert box["closed"]+ time.sleep(0.05)+ span = _one_finished_span(exporter)+ assert _finish_reasons(span) == ["incomplete"]+ assert "partial" in str(span.attributes)+++def test_bare_break_exports_the_span_when_the_wrapper_is_released():+ """Prem's documented call is a bare for-loop. break must not leak the span."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)+ box = {"closed": False}++ def run():+ traced = wrapper(+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),+ None,+ (),+ REQUEST_KWARGS,+ )+ for _ in traced:+ break++ run()+ gc.collect()++ assert box["closed"]+ span = _one_finished_span(exporter)+ assert span.status.status_code == StatusCode.OK+ assert _finish_reasons(span) == ["incomplete"]+ assert "partial" in str(span.attributes)+++def test_held_iter_only_stream_stays_open_until_released():+ """A kept reference must stay resumable. Finalizing on break would drop the tail."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)+ box = {"closed": False}++ stream = wrapper(+ lambda *a, **k: IterOnlyContainer(CHUNKS, box),+ None,+ (),+ REQUEST_KWARGS,+ )+ first = next(stream)++ assert first == CHUNKS[0]+ assert box["closed"] is False+ assert len(exporter.get_finished_spans()) == 0+ assert next(stream) == CHUNKS[1]++ del stream+ gc.collect()++ assert box["closed"]+ _one_finished_span(exporter)+++def test_stream_error_is_recorded_after_partial_text():+ """A failure from the generator stays on the span with the text already seen."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)+ box = {"closed": False}++ def run():+ traced = wrapper(+ lambda *a, **k: IterOnlyContainer(CHUNKS, box, raise_after=1),+ None,+ (),+ REQUEST_KWARGS,+ )+ with pytest.raises(RuntimeError, match="stream failed"):+ list(traced)++ run()+ gc.collect()++ assert box["closed"]+ span = _one_finished_span(exporter)+ assert span.status.status_code == StatusCode.ERROR+ assert _finish_reasons(span) == ["error"]+ assert "partial" in str(span.attributes)+++def test_attrs_chunks_contribute_content_and_response_id():+ """0.3 yields attrs objects. to_dict() is what puts the text on the span."""+ tracer, exporter = _tracer_with_exporter()+ wrapper = _factory(tracer)+ chunks = [AttrsChunk(chunk) for chunk in CHUNKS]+ box = {"closed": False}++ stream = wrapper(+ lambda *a, **k: IterOnlyContainer(chunks, box),+ None,+ (),+ REQUEST_KWARGS,+ )+ received = list(stream)++ assert received == chunks+ time.sleep(0.05)+ span = _one_finished_span(exporter)+ assert span.attributes.get(SemanticConvention.GEN_AI_RESPONSE_ID) == "chatcmpl-1"+ assert _finish_reasons(span) == ["stop"]+ assert "partial answer" in str(span.attributes)+ assert span.attributes.get(SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS) > 0+++def test_real_0_3_container_exposes_iter_only():+ """Lock the surface these tests stand in for, when that SDK is installed."""+ pytest.importorskip("premai")+ try:+ from premai.api.chat_completions.v1_chat_completions_create import (+ ChatCompletionResponseStreamContainer as container,+ )+ except ImportError:+ pytest.skip("premai.api is the 0.3 client; this install is newer")++ present = [+ name+ for name in ("__iter__", "__next__", "close", "__enter__", "__exit__")+ if name in vars(container)+ ]+ assert present == ["__iter__"]