Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(python-sdk): end OpenAI streaming spans on early stream close (#1455)

Amir Fathia346ed8c0f80
diff --git a/sdk/python/src/openlit/instrumentation/openai/async_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_openai.py
index b80c5c4a..05278225 100644
--- a/sdk/python/src/openlit/instrumentation/openai/async_openai.py
+++ b/sdk/python/src/openlit/instrumentation/openai/async_openai.py
@@ -80,13 +80,14 @@ def async_chat_completions(
self._server_address = server_address
self._server_port = server_port
self._event_provider = event_provider
+ self._streaming_response_processed = False
async def __aenter__(self):
await self.__wrapped__.__aenter__()
return self
async def __aexit__(self, exc_type, exc_value, traceback):
- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+ await self.close()
def __aiter__(self):
return self
@@ -95,27 +96,40 @@ def async_chat_completions(
"""Delegate attribute access to the wrapped object."""
return getattr(self.__wrapped__, name)
+ async def close(self):
+ """Close the wrapped stream and finalize the span if it has not ended yet."""
+ try:
+ await self.__wrapped__.close()
+ finally:
+ self._finalize_streaming_span()
+
+ def _finalize_streaming_span(self):
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ try:
+ with self._span:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=self._event_provider,
+ )
+ except Exception as e:
+ handle_exception(self._span, e)
+
async def __anext__(self):
try:
chunk = await self.__wrapped__.__anext__()
process_chat_chunk(self, chunk)
return chunk
except StopAsyncIteration:
- try:
- with self._span:
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=self._event_provider,
- )
- except Exception as e:
- handle_exception(self._span, e)
+ self._finalize_streaming_span()
raise
async def wrapper(wrapped, instance, args, kwargs):
@@ -308,13 +322,14 @@ def async_responses(
self._server_address = server_address
self._server_port = server_port
self._event_provider = event_provider
+ self._streaming_response_processed = False
async def __aenter__(self):
await self.__wrapped__.__aenter__()
return self
async def __aexit__(self, exc_type, exc_value, traceback):
- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)
+ await self.close()
def __aiter__(self):
return self
@@ -323,27 +338,40 @@ def async_responses(
"""Delegate attribute access to the wrapped object."""
return getattr(self.__wrapped__, name)
+ async def close(self):
+ """Close the wrapped stream and finalize the span if it has not ended yet."""
+ try:
+ await self.__wrapped__.close()
+ finally:
+ self._finalize_streaming_span()
+
+ def _finalize_streaming_span(self):
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ try:
+ with self._span:
+ process_streaming_response_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=self._event_provider,
+ )
+ except Exception as e:
+ handle_exception(self._span, e)
+
async def __anext__(self):
try:
chunk = await self.__wrapped__.__anext__()
process_response_chunk(self, chunk)
return chunk
except StopAsyncIteration:
- try:
- with self._span:
- process_streaming_response_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=self._event_provider,
- )
- except Exception as e:
- handle_exception(self._span, e)
+ self._finalize_streaming_span()
raise
async def wrapper(wrapped, instance, args, kwargs):
diff --git a/sdk/python/src/openlit/instrumentation/openai/openai.py b/sdk/python/src/openlit/instrumentation/openai/openai.py
index e88b37b3..02932b9a 100644
--- a/sdk/python/src/openlit/instrumentation/openai/openai.py
+++ b/sdk/python/src/openlit/instrumentation/openai/openai.py
@@ -80,13 +80,14 @@ def chat_completions(
self._server_address = server_address
self._server_port = server_port
self._event_provider = event_provider
+ self._streaming_response_processed = False
def __enter__(self):
self.__wrapped__.__enter__()
return self
def __exit__(self, exc_type, exc_value, traceback):
- self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+ self.close()
def __iter__(self):
return self
@@ -95,27 +96,40 @@ def chat_completions(
"""Delegate attribute access to the wrapped object."""
return getattr(self.__wrapped__, name)
+ def close(self):
+ """Close the wrapped stream and finalize the span if it has not ended yet."""
+ try:
+ self.__wrapped__.close()
+ finally:
+ self._finalize_streaming_span()
+
+ def _finalize_streaming_span(self):
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ try:
+ with self._span:
+ process_streaming_chat_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=self._event_provider,
+ )
+ except Exception as e:
+ handle_exception(self._span, e)
+
def __next__(self):
try:
chunk = self.__wrapped__.__next__()
process_chat_chunk(self, chunk)
return chunk
except StopIteration:
- try:
- with self._span:
- process_streaming_chat_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=self._event_provider,
- )
- except Exception as e:
- handle_exception(self._span, e)
+ self._finalize_streaming_span()
raise
def wrapper(wrapped, instance, args, kwargs):
@@ -310,13 +324,14 @@ def responses(
self._server_address = server_address
self._server_port = server_port
self._event_provider = event_provider
+ self._streaming_response_processed = False
def __enter__(self):
self.__wrapped__.__enter__()
return self
def __exit__(self, exc_type, exc_value, traceback):
- self.__wrapped__.__exit__(exc_type, exc_value, traceback)
+ self.close()
def __iter__(self):
return self
@@ -325,27 +340,40 @@ def responses(
"""Delegate attribute access to the wrapped object."""
return getattr(self.__wrapped__, name)
+ def close(self):
+ """Close the wrapped stream and finalize the span if it has not ended yet."""
+ try:
+ self.__wrapped__.close()
+ finally:
+ self._finalize_streaming_span()
+
+ def _finalize_streaming_span(self):
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ try:
+ with self._span:
+ process_streaming_response_response(
+ self,
+ pricing_info=pricing_info,
+ environment=environment,
+ application_name=application_name,
+ metrics=metrics,
+ capture_message_content=capture_message_content,
+ disable_metrics=disable_metrics,
+ version=version,
+ event_provider=self._event_provider,
+ )
+ except Exception as e:
+ handle_exception(self._span, e)
+
def __next__(self):
try:
chunk = self.__wrapped__.__next__()
process_response_chunk(self, chunk)
return chunk
except StopIteration:
- try:
- with self._span:
- process_streaming_response_response(
- self,
- pricing_info=pricing_info,
- environment=environment,
- application_name=application_name,
- metrics=metrics,
- capture_message_content=capture_message_content,
- disable_metrics=disable_metrics,
- version=version,
- event_provider=self._event_provider,
- )
- except Exception as e:
- handle_exception(self._span, e)
+ self._finalize_streaming_span()
raise
def wrapper(wrapped, instance, args, kwargs):
diff --git a/sdk/python/tests/test_openai_streaming_span_lifecycle.py b/sdk/python/tests/test_openai_streaming_span_lifecycle.py
new file mode 100644
index 00000000..14c0fb90
--- /dev/null
+++ b/sdk/python/tests/test_openai_streaming_span_lifecycle.py
@@ -0,0 +1,287 @@
+"""Tests that an early-exited OpenAI stream still ends and exports its span."""
+
+import pytest
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from openlit.instrumentation.openai.async_openai import async_chat_completions
+from openlit.instrumentation.openai.openai import chat_completions
+from openlit._config import OpenlitConfig
+
+
+def _tracer_and_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ tracer_provider = TracerProvider()
+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return tracer_provider.get_tracer("test-openai-stream-lifecycle"), exporter
+
+
+def _chat_kwargs():
+ return {
+ "model": "gpt-4o",
+ "messages": [{"role": "user", "content": "ping"}],
+ "stream": True,
+ }
+
+
+_CHUNK = {
+ "id": "chatcmpl-test",
+ "model": "gpt-4o",
+ "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}],
+}
+
+
+class FakeRawSyncStream:
+ """Minimal stand-in for openai._streaming.Stream: supports the context
+ manager + iterator protocol and an explicit close(), never driven to
+ exhaustion by this test. raise_on_close reproduces a client disconnect
+ or network error surfacing through the wrapped stream's own close(),
+ which must not prevent span finalization (review on #1455)."""
+
+ def __init__(self, chunks, raise_on_close=False):
+ self._chunks = list(chunks)
+ self.closed = False
+ self._raise_on_close = raise_on_close
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+ def __iter__(self):
+ return self
+
+ def __next__(self):
+ if not self._chunks:
+ raise StopIteration
+ return self._chunks.pop(0)
+
+ def close(self):
+ """Mark closed, matching openai._streaming.Stream.close()."""
+ self.closed = True
+ if self._raise_on_close:
+ raise RuntimeError("wrapped close failed")
+
+
+class FakeRawAsyncStream:
+ """Async twin of FakeRawSyncStream."""
+
+ def __init__(self, chunks, raise_on_close=False):
+ self._chunks = list(chunks)
+ self.closed = False
+ self._raise_on_close = raise_on_close
+
+ async def __aenter__(self):
+ return self
+
+ async def __aexit__(self, exc_type, exc_value, traceback):
+ await self.close()
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ if not self._chunks:
+ raise StopAsyncIteration
+ return self._chunks.pop(0)
+
+ async def close(self):
+ """Mark closed, matching openai._streaming.AsyncStream.close()."""
+ self.closed = True
+ if self._raise_on_close:
+ raise RuntimeError("wrapped close failed")
+
+
+def test_sync_stream_closed_via_context_manager_break_still_exports_span():
+ """An early `break` inside a `with ... as stream:` block must still end
+ and export the span, with whatever content had already streamed in."""
+
+ tracer, exporter = _tracer_and_exporter()
+ wrapper = chat_completions(
+ version="test-version",
+ environment="test-env",
+ application_name="test-app",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ raw_stream = FakeRawSyncStream([_CHUNK, _CHUNK])
+
+ def fake_create(*_args, **_kwargs):
+ return raw_stream
+
+ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())
+
+ with traced_stream as stream:
+ for _chunk in stream:
+ break
+
+ assert raw_stream.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].name == "chat gpt-4o"
+
+
+def test_sync_stream_explicit_close_still_exports_span():
+ """Calling .close() directly (no context manager) must also finalize."""
+
+ tracer, exporter = _tracer_and_exporter()
+ wrapper = chat_completions(
+ version="test-version",
+ environment="test-env",
+ application_name="test-app",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ raw_stream = FakeRawSyncStream([_CHUNK])
+
+ def fake_create(*_args, **_kwargs):
+ return raw_stream
+
+ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())
+ next(traced_stream)
+ traced_stream.close()
+
+ assert raw_stream.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+
+
+def test_sync_stream_full_consumption_exports_exactly_one_span():
+ """Full StopIteration consumption inside a `with` block must not
+ double-finalize (both __next__ and __exit__ fire)."""
+
+ tracer, exporter = _tracer_and_exporter()
+ wrapper = chat_completions(
+ version="test-version",
+ environment="test-env",
+ application_name="test-app",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ raw_stream = FakeRawSyncStream([_CHUNK])
+
+ def fake_create(*_args, **_kwargs):
+ return raw_stream
+
+ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())
+
+ with traced_stream as stream:
+ for _chunk in stream:
+ pass
+
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+
+
+def test_sync_stream_close_still_exports_span_when_wrapped_close_raises():
+ """If the wrapped stream's own close() raises (e.g. a network error on
+ disconnect), the span must still be finalized and exported rather than
+ silently dropped (Sourcery review on #1455: close() called the wrapped
+ close() before _finalize_streaming_span(), so a raise there skipped
+ finalization entirely)."""
+
+ tracer, exporter = _tracer_and_exporter()
+ wrapper = chat_completions(
+ version="test-version",
+ environment="test-env",
+ application_name="test-app",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ raw_stream = FakeRawSyncStream([_CHUNK], raise_on_close=True)
+
+ def fake_create(*_args, **_kwargs):
+ return raw_stream
+
+ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())
+ next(traced_stream)
+
+ with pytest.raises(RuntimeError):
+ traced_stream.close()
+
+ assert raw_stream.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+
+
+async def test_async_stream_closed_via_context_manager_break_still_exports_span():
+ """Async twin of the sync early-break test."""
+
+ tracer, exporter = _tracer_and_exporter()
+ wrapper = async_chat_completions(
+ version="test-version",
+ environment="test-env",
+ application_name="test-app",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ raw_stream = FakeRawAsyncStream([_CHUNK, _CHUNK])
+
+ async def fake_create(*_args, **_kwargs):
+ return raw_stream
+
+ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())
+
+ async with traced_stream as stream:
+ async for _chunk in stream:
+ break
+
+ assert raw_stream.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+
+
+async def test_async_stream_close_still_exports_span_when_wrapped_close_raises():
+ """Async twin of test_sync_stream_close_still_exports_span_when_wrapped_close_raises."""
+
+ tracer, exporter = _tracer_and_exporter()
+ wrapper = async_chat_completions(
+ version="test-version",
+ environment="test-env",
+ application_name="test-app",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=False,
+ metrics=None,
+ disable_metrics=True,
+ )
+
+ raw_stream = FakeRawAsyncStream([_CHUNK], raise_on_close=True)
+
+ async def fake_create(*_args, **_kwargs):
+ return raw_stream
+
+ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())
+ await anext(traced_stream)
+
+ with pytest.raises(RuntimeError):
+ await traced_stream.close()
+
+ assert raw_stream.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1