Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(python-sdk): end anthropic streaming spans on early exit (#1461)
fei757942ef01cd
diff --git a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.pyindex 6a4643c9..0fded86b 100644--- a/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py+++ b/sdk/python/src/openlit/instrumentation/anthropic/anthropic.py@@ -73,13 +73,20 @@ def messages(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = Falsedef __enter__(self):self.__wrapped__.__enter__()return selfdef __exit__(self, exc_type, exc_value, traceback):- self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ try:+ self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ finally:+ # Finalize on every exit: a break before exhaustion never+ # hits StopIteration, so the span would leak otherwise.+ if not exc_type:+ self._finalize_streaming_span()def __iter__(self):return self@@ -94,23 +101,42 @@ def messages(process_chunk(self, chunk)return chunkexcept StopIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=self._event_provider,- )- except Exception as e:- handle_exception(self._span, e)+ self._finalize_streaming_span()raise+ def _finalize_streaming_span(self):+ """Complete and end the span exactly once.++ Called on stream exhaustion and again from close()/manager exit;+ the flag keeps the double call a no-op so early exits that never+ see StopIteration still export the span instead of leaking it.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=self._event_provider,+ )+ except Exception as e:+ handle_exception(self._span, e)++ def close(self):+ """Close the wrapped stream and finalize the span if not ended."""+ try:+ self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()+def wrapper(wrapped, instance, args, kwargs):"""Wraps the Anthropic Messages.create call.@@ -256,6 +282,7 @@ def messages_stream(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = Falsedef __iter__(self):return self@@ -304,23 +331,42 @@ def messages_stream(process_chunk(self, chunk)return chunkexcept StopIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=self._event_provider,- )- except Exception as e:- handle_exception(self._span, e)+ self._finalize_streaming_span()raise+ def _finalize_streaming_span(self):+ """Complete and end the span exactly once.++ Called on stream exhaustion and again from close()/manager exit;+ the flag keeps the double call a no-op so early exits that never+ see StopIteration still export the span instead of leaking it.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=self._event_provider,+ )+ except Exception as e:+ handle_exception(self._span, e)++ def close(self):+ """Close the wrapped stream and finalize the span if not ended."""+ try:+ self.__wrapped__.close()+ finally:+ self._finalize_streaming_span()+class TracedMessageStreamManager:"""Wrapper for Anthropic MessageStreamManager to instrument the 'with' block.@@ -344,6 +390,7 @@ def messages_stream(self._server_port = server_portself._event_provider = event_providerself._token = None+ self._traced_stream = Nonedef __enter__(self):"""@@ -354,7 +401,7 @@ def messages_stream(stream = self._original_manager.__enter__()- return TracedMessageStream(+ self._traced_stream = TracedMessageStream(stream,self._span,self._span_name,@@ -363,6 +410,7 @@ def messages_stream(self._server_port,self._event_provider,)+ return self._traced_streamdef __exit__(self, exc_type, exc_val, exc_tb):"""@@ -376,6 +424,12 @@ def messages_stream(handle_exception(self._span, exc_val)if self._span.is_recording():self._span.end()+ else:+ # Early exit from the `with` block (e.g. a break before the+ # stream is exhausted): __next__ never hit StopIteration, so+ # finalize the span here or it is never exported (#1454).+ if self._traced_stream is not None:+ self._traced_stream._finalize_streaming_span()return self._original_manager.__exit__(exc_type, exc_val, exc_tb)diff --git a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.pyindex c7bf7257..06c02bc3 100644--- a/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py+++ b/sdk/python/src/openlit/instrumentation/anthropic/async_anthropic.py@@ -74,13 +74,20 @@ def async_messages(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = Falseasync def __aenter__(self):await self.__wrapped__.__aenter__()return selfasync def __aexit__(self, exc_type, exc_value, traceback):- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ try:+ await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ finally:+ # Finalize on every exit: breaking out of the async loop+ # never hits StopAsyncIteration, so the span would leak.+ if not exc_type:+ self._finalize_streaming_span()def __aiter__(self):return self@@ -95,23 +102,30 @@ def async_messages(process_chunk(self, chunk)return chunkexcept StopAsyncIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=self._event_provider,- )- except Exception as e:- handle_exception(self._span, e)+ self._finalize_streaming_span()raise+ def _finalize_streaming_span(self):+ """Complete and end the span exactly once (see the sync twin)."""+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=self._event_provider,+ )+ except Exception as e:+ handle_exception(self._span, e)+async def wrapper(wrapped, instance, args, kwargs):"""Wraps the Anthropic AsyncMessages.create call.@@ -258,6 +272,7 @@ def async_messages_stream(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = Falsedef __aiter__(self):return self@@ -306,23 +321,30 @@ def async_messages_stream(process_chunk(self, chunk)return chunkexcept StopAsyncIteration:- try:- with self._span:- process_streaming_chat_response(- self,- pricing_info=pricing_info,- environment=environment,- application_name=application_name,- metrics=metrics,- capture_message_content=capture_message_content,- disable_metrics=disable_metrics,- version=version,- event_provider=self._event_provider,- )- except Exception as e:- handle_exception(self._span, e)+ self._finalize_streaming_span()raise+ def _finalize_streaming_span(self):+ """Complete and end the span exactly once (see the sync twin)."""+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ try:+ with self._span:+ process_streaming_chat_response(+ self,+ pricing_info=pricing_info,+ environment=environment,+ application_name=application_name,+ metrics=metrics,+ capture_message_content=capture_message_content,+ disable_metrics=disable_metrics,+ version=version,+ event_provider=self._event_provider,+ )+ except Exception as e:+ handle_exception(self._span, e)+class TracedAsyncMessageStreamManager:"""Wrapper for Anthropic AsyncMessageStreamManager to instrument the 'async with' block.@@ -346,6 +368,7 @@ def async_messages_stream(self._server_port = server_portself._event_provider = event_providerself._token = None+ self._traced_stream = Noneself._attaching_task = Noneasync def __aenter__(self):@@ -358,7 +381,7 @@ def async_messages_stream(stream = await self._original_manager.__aenter__()- return TracedAsyncMessageStream(+ self._traced_stream = TracedAsyncMessageStream(stream,self._span,self._span_name,@@ -367,6 +390,7 @@ def async_messages_stream(self._server_port,self._event_provider,)+ return self._traced_streamasync def __aexit__(self, exc_type, exc_val, exc_tb):"""@@ -381,6 +405,12 @@ def async_messages_stream(handle_exception(self._span, exc_val)if self._span.is_recording():self._span.end()+ else:+ # Early exit from the `async with` block: __anext__ never hit+ # StopAsyncIteration, so finalize the span here or it is+ # never exported (#1454).+ if self._traced_stream is not None:+ self._traced_stream._finalize_streaming_span()return await self._original_manager.__aexit__(exc_type, exc_val, exc_tb)diff --git a/sdk/python/tests/test_anthropic_stream_span_lifecycle.py b/sdk/python/tests/test_anthropic_stream_span_lifecycle.pynew file mode 100644index 00000000..9ddcde77--- /dev/null+++ b/sdk/python/tests/test_anthropic_stream_span_lifecycle.py@@ -0,0 +1,191 @@+# pylint: disable=protected-access+"""Regression tests: the Anthropic streaming wrappers must end their span on+every exit path.++`TracedMessageStreamManager.__exit__` ended the span only when the `with`+block raised; leaving the block early (a `break` before `StopIteration`, or a+bare `stream.close()`) left the span recording forever, so it was never+exported and its telemetry was lost — the Anthropic instance of the early-+close streaming bug filed for OpenAI in #1454. These tests drive the real+`messages_stream`/`async_messages_stream` wrapper factories with synthetic+managers and event streams shaped like the anthropic SDK's, and assert the+span ends exactly once on each exit path.+"""++import time+from collections.abc import AsyncIterator++from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++from openlit._config import OpenlitConfig+from openlit.instrumentation.anthropic import anthropic as sync_mod+from openlit.instrumentation.anthropic import async_anthropic as async_mod++REQUEST_KWARGS = {"model": "claude-3-5-sonnet-latest"}++EVENTS = [+ {+ "type": "message_start",+ "message": {+ "id": "msg_01",+ "model": "claude-3-5-sonnet-latest",+ "role": "assistant",+ "usage": {"input_tokens": 10},+ },+ },+ {+ "type": "content_block_start",+ "index": 0,+ "content_block": {"type": "text", "text": ""},+ },+ {+ "type": "content_block_delta",+ "index": 0,+ "delta": {"type": "text_delta", "text": "partial answer"},+ },+ {"type": "content_block_stop", "index": 0},+ {+ "type": "message_delta",+ "delta": {"stop_reason": "end_turn"},+ "usage": {"output_tokens": 5},+ },+ {"type": "message_stop"},+]+++def _tracer_with_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _factory(tracer, *, is_async):+ mod = async_mod if is_async else sync_mod+ make = mod.async_messages_stream if is_async else mod.messages_stream+ return make(+ version="test",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=True,+ metrics=None,+ disable_metrics=True,+ )+++class FakeRawStream:+ """Sync/async event stream shaped like anthropic's MessageStream."""++ def __init__(self):+ self._it = iter(EVENTS)+ self.closed = False++ def __iter__(self):+ return self++ def __next__(self):+ return next(self._it)++ def __aiter__(self):+ return self++ async def __anext__(self):+ try:+ return next(self._it)+ except StopIteration:+ raise StopAsyncIteration from None++ def close(self):+ self.closed = True+++class FakeManager:+ """Context manager shaped like anthropic's MessageStreamManager."""++ def __init__(self, stream):+ self._stream = stream++ def __enter__(self):+ return self._stream++ def __exit__(self, *exc):+ return False++ async def __aenter__(self):+ return self._stream++ async def __aexit__(self, *exc):+ return False+++def test_sync_early_break_inside_with_ends_span():+ tracer, exporter = _tracer_with_exporter()+ factory = _factory(tracer, is_async=False)++ manager = factory(lambda *a, **k: FakeManager(FakeRawStream()), None, (), REQUEST_KWARGS)+ with manager as stream:+ next(stream) # consume one event, then leave the block early++ time.sleep(0.1)+ spans = exporter.get_finished_spans()+ assert len(spans) == 1, "early exit must still end and export the span"+++def test_sync_full_consumption_exports_exactly_one_span():+ tracer, exporter = _tracer_with_exporter()+ factory = _factory(tracer, is_async=False)++ manager = factory(lambda *a, **k: FakeManager(FakeRawStream()), None, (), REQUEST_KWARGS)+ with manager as stream:+ for _ in stream:+ pass++ time.sleep(0.1)+ assert len(exporter.get_finished_spans()) == 1+++def test_sync_close_finalizes_span():+ tracer, exporter = _tracer_with_exporter()+ factory = _factory(tracer, is_async=False)++ manager = factory(lambda *a, **k: FakeManager(FakeRawStream()), None, (), REQUEST_KWARGS)+ with manager as stream:+ next(stream)+ stream.close()++ time.sleep(0.1)+ spans = exporter.get_finished_spans()+ assert len(spans) == 1, "close() must finalize the span exactly once"+++async def test_async_early_break_inside_with_ends_span():+ tracer, exporter = _tracer_with_exporter()+ factory = _factory(tracer, is_async=True)++ manager = factory(lambda *a, **k: FakeManager(FakeRawStream()), None, (), REQUEST_KWARGS)+ async with manager as stream:+ await stream.__anext__()++ time.sleep(0.1)+ spans = exporter.get_finished_spans()+ assert len(spans) == 1, "early async exit must still end and export the span"+++async def test_async_full_consumption_exports_exactly_one_span():+ tracer, exporter = _tracer_with_exporter()+ factory = _factory(tracer, is_async=True)++ manager = factory(lambda *a, **k: FakeManager(FakeRawStream()), None, (), REQUEST_KWARGS)+ async with manager as stream:+ async for _ in stream:+ pass++ time.sleep(0.1)+ assert len(exporter.get_finished_spans()) == 1