Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(python-sdk): end OpenAI streaming spans on early stream close (#1455)
Amir Fathia346ed8c0f80
diff --git a/sdk/python/src/openlit/instrumentation/openai/async_openai.py b/sdk/python/src/openlit/instrumentation/openai/async_openai.pyindex b80c5c4a..05278225 100644--- a/sdk/python/src/openlit/instrumentation/openai/async_openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/async_openai.py@@ -80,13 +80,14 @@ def async_chat_completions(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = Falseasync def __aenter__(self):await self.__wrapped__.__aenter__()return selfasync def __aexit__(self, exc_type, exc_value, traceback):- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ await self.close()def __aiter__(self):return self@@ -95,27 +96,40 @@ def async_chat_completions("""Delegate attribute access to the wrapped object."""return getattr(self.__wrapped__, name)+ async def close(self):+ """Close the wrapped stream and finalize the span if it has not ended yet."""+ try:+ await self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()++ def _finalize_streaming_span(self):+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=self._event_provider,+ )+ except Exception as e:+ handle_exception(self._span, e)+async def __anext__(self):try:chunk = await self.__wrapped__.__anext__()process_chat_chunk(self, chunk)return chunkexcept StopAsyncIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=self._event_provider,- )- except Exception as e:- handle_exception(self._span, e)+ self._finalize_streaming_span()raiseasync def wrapper(wrapped, instance, args, kwargs):@@ -308,13 +322,14 @@ def async_responses(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = Falseasync def __aenter__(self):await self.__wrapped__.__aenter__()return selfasync def __aexit__(self, exc_type, exc_value, traceback):- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ await self.close()def __aiter__(self):return self@@ -323,27 +338,40 @@ def async_responses("""Delegate attribute access to the wrapped object."""return getattr(self.__wrapped__, name)+ async def close(self):+ """Close the wrapped stream and finalize the span if it has not ended yet."""+ try:+ await self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()++ def _finalize_streaming_span(self):+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_response_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=self._event_provider,+ )+ except Exception as e:+ handle_exception(self._span, e)+async def __anext__(self):try:chunk = await self.__wrapped__.__anext__()process_response_chunk(self, chunk)return chunkexcept StopAsyncIteration:- try:- with self._span:- process_streaming_response_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=self._event_provider,- )- except Exception as e:- handle_exception(self._span, e)+ self._finalize_streaming_span()raiseasync def wrapper(wrapped, instance, args, kwargs):diff --git a/sdk/python/src/openlit/instrumentation/openai/openai.py b/sdk/python/src/openlit/instrumentation/openai/openai.pyindex e88b37b3..02932b9a 100644--- a/sdk/python/src/openlit/instrumentation/openai/openai.py+++ b/sdk/python/src/openlit/instrumentation/openai/openai.py@@ -80,13 +80,14 @@ def chat_completions(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = Falsedef __enter__(self):self.__wrapped__.__enter__()return selfdef __exit__(self, exc_type, exc_value, traceback):- self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ self.close()def __iter__(self):return self@@ -95,27 +96,40 @@ def chat_completions("""Delegate attribute access to the wrapped object."""return getattr(self.__wrapped__, name)+ def close(self):+ """Close the wrapped stream and finalize the span if it has not ended yet."""+ try:+ self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()++ def _finalize_streaming_span(self):+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=self._event_provider,+ )+ except Exception as e:+ handle_exception(self._span, e)+def __next__(self):try:chunk = self.__wrapped__.__next__()process_chat_chunk(self, chunk)return chunkexcept StopIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=self._event_provider,- )- except Exception as e:- handle_exception(self._span, e)+ self._finalize_streaming_span()raisedef wrapper(wrapped, instance, args, kwargs):@@ -310,13 +324,14 @@ def responses(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = Falsedef __enter__(self):self.__wrapped__.__enter__()return selfdef __exit__(self, exc_type, exc_value, traceback):- self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ self.close()def __iter__(self):return self@@ -325,27 +340,40 @@ def responses("""Delegate attribute access to the wrapped object."""return getattr(self.__wrapped__, name)+ def close(self):+ """Close the wrapped stream and finalize the span if it has not ended yet."""+ try:+ self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()++ def _finalize_streaming_span(self):+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_response_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=self._event_provider,+ )+ except Exception as e:+ handle_exception(self._span, e)+def __next__(self):try:chunk = self.__wrapped__.__next__()process_response_chunk(self, chunk)return chunkexcept StopIteration:- try:- with self._span:- process_streaming_response_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=self._event_provider,- )- except Exception as e:- handle_exception(self._span, e)+ self._finalize_streaming_span()raisedef wrapper(wrapped, instance, args, kwargs):diff --git a/sdk/python/tests/test_openai_streaming_span_lifecycle.py b/sdk/python/tests/test_openai_streaming_span_lifecycle.pynew file mode 100644index 00000000..14c0fb90--- /dev/null+++ b/sdk/python/tests/test_openai_streaming_span_lifecycle.py@@ -0,0 +1,287 @@+"""Tests that an early-exited OpenAI stream still ends and exports its span."""++import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter++from openlit.instrumentation.openai.async_openai import async_chat_completions+from openlit.instrumentation.openai.openai import chat_completions+from openlit._config import OpenlitConfig+++def _tracer_and_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))+ return tracer_provider.get_tracer("test-openai-stream-lifecycle"), exporter+++def _chat_kwargs():+ return {+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "ping"}],+ "stream": True,+ }+++_CHUNK = {+ "id": "chatcmpl-test",+ "model": "gpt-4o",+ "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}],+}+++class FakeRawSyncStream:+ """Minimal stand-in for openai._streaming.Stream: supports the context+ manager + iterator protocol and an explicit close(), never driven to+ exhaustion by this test. raise_on_close reproduces a client disconnect+ or network error surfacing through the wrapped stream's own close(),+ which must not prevent span finalization (review on #1455)."""++ def __init__(self, chunks, raise_on_close=False):+ self._chunks = list(chunks)+ self.closed = False+ self._raise_on_close = raise_on_close++ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self.close()++ def __iter__(self):+ return self++ def __next__(self):+ if not self._chunks:+ raise StopIteration+ return self._chunks.pop(0)++ def close(self):+ """Mark closed, matching openai._streaming.Stream.close()."""+ self.closed = True+ if self._raise_on_close:+ raise RuntimeError("wrapped close failed")+++class FakeRawAsyncStream:+ """Async twin of FakeRawSyncStream."""++ def __init__(self, chunks, raise_on_close=False):+ self._chunks = list(chunks)+ self.closed = False+ self._raise_on_close = raise_on_close++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ await self.close()++ def __aiter__(self):+ return self++ async def __anext__(self):+ if not self._chunks:+ raise StopAsyncIteration+ return self._chunks.pop(0)++ async def close(self):+ """Mark closed, matching openai._streaming.AsyncStream.close()."""+ self.closed = True+ if self._raise_on_close:+ raise RuntimeError("wrapped close failed")+++def test_sync_stream_closed_via_context_manager_break_still_exports_span():+ """An early `break` inside a `with ... as stream:` block must still end+ and export the span, with whatever content had already streamed in."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = chat_completions(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ raw_stream = FakeRawSyncStream([_CHUNK, _CHUNK])++ def fake_create(*_args, **_kwargs):+ return raw_stream++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())++ with traced_stream as stream:+ for _chunk in stream:+ break++ assert raw_stream.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].name == "chat gpt-4o"+++def test_sync_stream_explicit_close_still_exports_span():+ """Calling .close() directly (no context manager) must also finalize."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = chat_completions(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ raw_stream = FakeRawSyncStream([_CHUNK])++ def fake_create(*_args, **_kwargs):+ return raw_stream++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())+ next(traced_stream)+ traced_stream.close()++ assert raw_stream.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+++def test_sync_stream_full_consumption_exports_exactly_one_span():+ """Full StopIteration consumption inside a `with` block must not+ double-finalize (both __next__ and __exit__ fire)."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = chat_completions(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ raw_stream = FakeRawSyncStream([_CHUNK])++ def fake_create(*_args, **_kwargs):+ return raw_stream++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())++ with traced_stream as stream:+ for _chunk in stream:+ pass++ spans = exporter.get_finished_spans()+ assert len(spans) == 1+++def test_sync_stream_close_still_exports_span_when_wrapped_close_raises():+ """If the wrapped stream's own close() raises (e.g. a network error on+ disconnect), the span must still be finalized and exported rather than+ silently dropped (Sourcery review on #1455: close() called the wrapped+ close() before _finalize_streaming_span(), so a raise there skipped+ finalization entirely)."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = chat_completions(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ raw_stream = FakeRawSyncStream([_CHUNK], raise_on_close=True)++ def fake_create(*_args, **_kwargs):+ return raw_stream++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())+ next(traced_stream)++ with pytest.raises(RuntimeError):+ traced_stream.close()++ assert raw_stream.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+++async def test_async_stream_closed_via_context_manager_break_still_exports_span():+ """Async twin of the sync early-break test."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = async_chat_completions(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ raw_stream = FakeRawAsyncStream([_CHUNK, _CHUNK])++ async def fake_create(*_args, **_kwargs):+ return raw_stream++ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())++ async with traced_stream as stream:+ async for _chunk in stream:+ break++ assert raw_stream.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+++async def test_async_stream_close_still_exports_span_when_wrapped_close_raises():+ """Async twin of test_sync_stream_close_still_exports_span_when_wrapped_close_raises."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = async_chat_completions(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=None,+ disable_metrics=True,+ )++ raw_stream = FakeRawAsyncStream([_CHUNK], raise_on_close=True)++ async def fake_create(*_args, **_kwargs):+ return raw_stream++ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())+ await anext(traced_stream)++ with pytest.raises(RuntimeError):+ await traced_stream.close()++ assert raw_stream.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1