Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(bedrock): end converse_stream span on early close (#1608)
RichardoMu79520db501cc
diff --git a/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.pyindex de51064d..b7634bfb 100644--- a/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py+++ b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py@@ -132,7 +132,7 @@ def converse_stream(Generates a telemetry wrapper for AWS Bedrock converse_stream calls."""- class TracedSyncStream:+ class TracedSyncStream: # pylint: disable=too-many-instance-attributes"""Wrapper for streaming responses to collect telemetry."""@@ -149,11 +149,16 @@ def converse_stream(**args,):self.__wrapped_response = wrapped_response- # Extract the actual stream iterator from the response+ # Keep the botocore EventStream intact. Its __iter__ is a generator+ # function, so calling iter() here would open a second generator+ # over the one-shot event body before the caller reads+ # response["stream"].if isinstance(wrapped_response, dict) and "stream" in wrapped_response:- self.__wrapped_stream = iter(wrapped_response["stream"])+ self._event_stream = wrapped_response["stream"]else:- self.__wrapped_stream = iter(wrapped_response)+ self._event_stream = wrapped_response+ self._iterator = None+ self._closed = Falseself._span = spanself._span_name = span_name@@ -178,17 +183,21 @@ def converse_stream(self._server_address = server_addressself._server_port = server_portself._event_provider = event_provider+ self._streaming_response_processed = False+ self._stream_error = Nonedef __enter__(self):- if hasattr(self.__wrapped_stream, "__enter__"):- self.__wrapped_stream.__enter__()+ # The boto3 response dict is not a context manager. Callers that+ # use ``with client.converse_stream()`` are using this wrapper.return selfdef __exit__(self, exc_type, exc_value, traceback):- if hasattr(self.__wrapped_stream, "__exit__"):- self.__wrapped_stream.__exit__(exc_type, exc_value, traceback)+ # A raise from user code is a cancel. A failure raised by the+ # event stream was stored on _stream_error in __next__.+ self.close()def __iter__(self):+ self._ensure_iterator()return selfdef __getattr__(self, name):@@ -196,25 +205,108 @@ def converse_stream(return getattr(self.__wrapped_response, name)def get(self, key, default=None):- """Delegate get method to the wrapped response if its a dict."""+ """Return this wrapper for ``stream``; delegate every other key.++ boto3 callers do ``for event in response["stream"]``. The raw+ EventStream under that key never enters __next__, so the span+ would stay open and the chunks would never be recorded.+ """+ if key == "stream":+ return selfif isinstance(self.__wrapped_response, dict):return self.__wrapped_response.get(key, default)return getattr(self.__wrapped_response, key, default)def __getitem__(self, key):- """Delegate item access to the wrapped response if its a dict."""+ """Return this wrapper for ``stream``; delegate every other key."""+ if key == "stream":+ return selfif isinstance(self.__wrapped_response, dict):return self.__wrapped_response[key]return getattr(self.__wrapped_response, key)+ def close(self):+ """Release the HTTP body and end the span once.++ ``EventStream.__iter__`` returns a generator. That generator's+ close() does not call ``EventStream.close()``, and+ ``StreamingBody.__del__`` does not close the socket.+ """+ try:+ self._close_body()+ finally:+ self._finalize_streaming_span(ended_cleanly=False)++ def __del__(self):+ """End a span abandoned by a bare ``for`` loop.++ ``break`` does not call close() on this iterator. Nothing else+ closes the botocore stream, so dropping the response has to.+ """+ try:+ self.close()+ except Exception:+ pass++ def _ensure_iterator(self):+ if self._iterator is None:+ self._iterator = iter(self._event_stream)+ return self._iterator++ def _close_body(self):+ if self._closed:+ return+ self._closed = True+ iterator = self._iterator+ event_stream = self._event_stream+ # The generator and the EventStream are different objects.+ # Closing only the generator leaves the HTTP connection open.+ if iterator is not None and iterator is not event_stream:+ iterator_close = getattr(iterator, "close", None)+ if iterator_close is not None:+ try:+ iterator_close()+ except Exception:+ pass+ stream_close = getattr(event_stream, "close", None)+ if stream_close is not None:+ try:+ stream_close()+ except Exception:+ pass+def __next__(self):try:- chunk = next(self.__wrapped_stream)- process_chunk(self, chunk)- return chunk+ chunk = next(self._ensure_iterator())except StopIteration:- try:- with self._span:+ self._finalize_streaming_span(ended_cleanly=True)+ self._close_body()+ raise+ except Exception as exc:+ self._stream_error = exc+ raise+ process_chunk(self, chunk)+ return chunk++ def _finalize_streaming_span(self, ended_cleanly):+ """End the span once.++ ``ended_cleanly`` is True only for StopIteration. An empty finish+ reason is otherwise a cancel. A failure raised by the event stream+ is recorded after attributes are set, so partial text stays on the+ span and the status is ERROR rather than the OK that+ ``process_streaming_chat_response`` writes when nothing failed.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ if self._stream_error is not None and not self._finish_reason:+ self._finish_reason = "error"+ elif not ended_cleanly and not self._finish_reason:+ self._finish_reason = "incomplete"+ try:+ with self._span:+ try:process_streaming_chat_response(self,pricing_info=pricing_info,@@ -226,11 +318,11 @@ def converse_stream(version=version,event_provider=self._event_provider,)-- except Exception as e:- handle_exception(self._span, e)-- raise+ finally:+ if self._stream_error is not None:+ handle_exception(self._span, self._stream_error)+ except Exception as e:+ handle_exception(self._span, e)def wrapper(wrapped, instance, args, kwargs):"""diff --git a/sdk/python/src/openlit/instrumentation/bedrock/utils.py b/sdk/python/src/openlit/instrumentation/bedrock/utils.pyindex 26642143..c7619c41 100644--- a/sdk/python/src/openlit/instrumentation/bedrock/utils.py+++ b/sdk/python/src/openlit/instrumentation/bedrock/utils.py@@ -602,7 +602,12 @@ def common_chat_logic(except Exception as e:logger.warning("Failed to emit inference event: %s", e, exc_info=True)- scope._span.set_status(Status(StatusCode.OK))+ # OK is sticky in the OTel SDK: a later set_status(ERROR) is ignored.+ # Stream failures set scope._stream_error before this runs, then+ # handle_exception records ERROR once these attributes are written.+ # Non-streaming scopes never set _stream_error, so they still record OK.+ if getattr(scope, "_stream_error", None) is None:+ scope._span.set_status(Status(StatusCode.OK))# Record metricsif not disable_metrics:diff --git a/sdk/python/tests/test_bedrock_stream_span_lifecycle.py b/sdk/python/tests/test_bedrock_stream_span_lifecycle.pynew file mode 100644index 00000000..e33abab0--- /dev/null+++ b/sdk/python/tests/test_bedrock_stream_span_lifecycle.py@@ -0,0 +1,325 @@+# pylint: disable=protected-access, too-few-public-methods, not-context-manager+"""Bedrock converse_stream spans must export on the path boto3 callers use.++``converse_stream`` returns a dict. Production code iterates+``response["stream"]`` and may call ``close()`` on that object. botocore's+``EventStream.__iter__`` is a generator function: ``iter(stream)`` is not the+object ``close()`` releases the HTTP body on, and a bare ``for`` plus ``break``+never calls ``close()``.++The fake stream below has that shape. A self-iterating iterator would hide+both the missed span and the leaked connection.+"""++import gc++import pytest+from opentelemetry import context as context_api+from opentelemetry import trace as trace_api+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)+from opentelemetry.trace import StatusCode++from openlit._config import OpenlitConfig+from openlit.instrumentation.bedrock.bedrock import converse_stream+from openlit.semcov import SemanticConvention++REQUEST_KWARGS = {"modelId": "amazon.titan-text-express-v1"}+++def _delta(text):+ return {+ "contentBlockDelta": {+ "delta": {"text": text},+ "contentBlockIndex": 0,+ }+ }+++def _terminal_events():+ return [+ {"messageStop": {"stopReason": "end_turn"}},+ {+ "metadata": {+ "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15},+ "metrics": {"latencyMs": 42},+ }+ },+ ]+++class FakeEventStream:+ """botocore EventStream: ``__iter__`` yields, ``close()`` closes the body."""++ def __init__(self, events, raise_after=None):+ self._events = list(events)+ self._raise_after = raise_after+ self.closed = False+ self.iter_calls = 0++ def __iter__(self):+ self.iter_calls += 1+ for index, event in enumerate(self._events):+ if self.closed:+ return+ if self._raise_after is not None and index >= self._raise_after:+ raise RuntimeError("stream failed")+ yield event++ def close(self):+ """Record the HTTP-body close. Generator.close() must not be enough."""+ self.closed = True+++class FakeClient:+ """Minimal stand-in for a bedrock-runtime client."""++ def __init__(self, stream):+ self.stream = stream+ self.calls = []++ def converse_stream(self, **kwargs):+ """Return a boto3-shaped streaming response."""+ self.calls.append(kwargs)+ return {"stream": self.stream, "ResponseMetadata": {"RequestId": "req-1"}}+++def _tracer_and_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _client(tracer, events, raise_after=None, capture_message_content=True):+ wrapper = converse_stream(+ version="test",+ environment="test",+ application_name="test",+ tracer=tracer,+ pricing_info={},+ capture_message_content=capture_message_content,+ metrics=None,+ disable_metrics=True,+ )+ fake = FakeEventStream(events, raise_after=raise_after)+ raw = FakeClient(fake)+ client = wrapper(+ lambda *args, **kwargs: raw,+ None,+ (),+ {"service_name": "bedrock-runtime"},+ )+ return client, fake+++def _finish_reasons(span):+ return tuple(span.attributes[SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON])+++def _output(span):+ return span.attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]+++def _assert_tokens(span):+ assert span.attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 10+ assert span.attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 5+++def test_constructing_the_wrapper_does_not_read_or_close_the_stream():+ """``iter()`` in ``__init__`` would open a second generator over the body."""++ tracer, _exporter = _tracer_and_exporter()+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])++ response = client.converse_stream(**REQUEST_KWARGS)+ stream = response.get("stream")++ assert stream is response+ assert response["ResponseMetadata"]["RequestId"] == "req-1"+ assert fake.iter_calls == 0+ assert fake.closed is False+++def test_break_on_response_stream_ends_span_and_closes_body():+ """The AWS sample loop must export one partial span and close the body."""++ tracer, exporter = _tracer_and_exporter()+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])++ response = client.converse_stream(**REQUEST_KWARGS)+ for event in response["stream"]:+ assert "partial" in event["contentBlockDelta"]["delta"]["text"]+ break++ del response+ gc.collect()++ assert fake.closed+ assert fake.iter_calls == 1+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("incomplete",)+ assert "partial" in _output(spans[0])+ assert "answer" not in _output(spans[0])+++def test_get_stream_break_inside_with_ends_span():+ """``with`` plus ``response.get('stream')`` is the same iterator and span."""++ tracer, exporter = _tracer_and_exporter()+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])++ with client.converse_stream(**REQUEST_KWARGS) as response:+ stream = response.get("stream")+ assert stream is response+ for _event in stream:+ break++ assert fake.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("incomplete",)+ assert "partial" in _output(spans[0])+ assert "answer" not in _output(spans[0])+++def test_held_stream_stays_open_until_released():+ """Keeping the response must leave the next event readable."""++ tracer, exporter = _tracer_and_exporter()+ client, fake = _client(+ tracer, [_delta("one"), _delta("two"), _delta("three")]+ )++ response = client.converse_stream(**REQUEST_KWARGS)+ stream = response["stream"]+ for event in stream:+ assert event["contentBlockDelta"]["delta"]["text"] == "one"+ break++ assert not exporter.get_finished_spans()+ assert fake.closed is False+ assert next(stream)["contentBlockDelta"]["delta"]["text"] == "two"++ del response+ del stream+ gc.collect()++ assert fake.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert _finish_reasons(spans[0]) == ("incomplete",)+ output = _output(spans[0])+ assert "one" in output+ assert "two" in output+ assert "three" not in output+++def test_full_consumption_of_response_stream_exports_exactly_one_span():+ """StopIteration then close() must not emit a second span or rewrite the reason."""++ tracer, exporter = _tracer_and_exporter()+ events = [_delta("partial"), _delta(" answer"), *_terminal_events()]+ client, fake = _client(tracer, events)++ response = client.converse_stream(**REQUEST_KWARGS)+ for _event in response["stream"]:+ pass+ for _event in response["stream"]:+ pass+ response["stream"].close()++ assert fake.closed+ assert fake.iter_calls == 1+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("end_turn",)+ assert "partial answer" in _output(spans[0])+ _assert_tokens(spans[0])+++def test_close_on_response_stream_finalizes_once():+ """``response['stream'].close()`` ends the span and the HTTP body."""++ tracer, exporter = _tracer_and_exporter()+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])++ response = client.converse_stream(**REQUEST_KWARGS)+ next(response["stream"])+ response["stream"].close()+ response.close()++ assert fake.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("incomplete",)+ assert "partial" in _output(spans[0])+ assert "answer" not in _output(spans[0])+++def test_caller_exception_inside_with_stays_ok():+ """A raise from user code is a cancel, not a Bedrock failure."""++ tracer, exporter = _tracer_and_exporter()+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])++ with pytest.raises(ValueError, match="caller"):+ with client.converse_stream(**REQUEST_KWARGS) as response:+ for _event in response["stream"]:+ raise ValueError("caller")++ assert fake.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("incomplete",)+ assert SemanticConvention.ERROR_TYPE not in spans[0].attributes+ assert all(event.name != "exception" for event in spans[0].events)+ assert "partial" in _output(spans[0])+ assert "answer" not in _output(spans[0])+++def test_stream_error_inside_with_marks_span_error():+ """A failure from the event stream is a failed span, with text already seen."""++ tracer, exporter = _tracer_and_exporter()+ client, fake = _client(+ tracer, [_delta("partial"), _delta("answer")], raise_after=1+ )++ with pytest.raises(RuntimeError, match="stream failed"):+ with client.converse_stream(**REQUEST_KWARGS) as response:+ for _event in response["stream"]:+ pass++ assert fake.closed+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.ERROR+ assert spans[0].attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"+ assert _finish_reasons(spans[0]) == ("error",)+ assert "partial" in _output(spans[0])+ assert "answer" not in _output(spans[0])+++def test_context_detached_after_early_break():+ """Finalizing the span must not leave it attached to the OTel context."""++ tracer, _exporter = _tracer_and_exporter()+ client, _fake = _client(tracer, [_delta("partial")])++ with client.converse_stream(**REQUEST_KWARGS) as response:+ next(response["stream"])++ assert (+ trace_api.get_current_span(context_api.get_current()) is trace_api.INVALID_SPAN+ )