Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

fix(bedrock): end converse_stream span on early close (#1608)

RichardoMu79520db501cc
diff --git a/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py
index de51064d..b7634bfb 100644
--- a/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py
+++ b/sdk/python/src/openlit/instrumentation/bedrock/bedrock.py
@@ -132,7 +132,7 @@ def converse_stream(
Generates a telemetry wrapper for AWS Bedrock converse_stream calls.
"""
- class TracedSyncStream:
+ class TracedSyncStream: # pylint: disable=too-many-instance-attributes
"""
Wrapper for streaming responses to collect telemetry.
"""
@@ -149,11 +149,16 @@ def converse_stream(
**args,
):
self.__wrapped_response = wrapped_response
- # Extract the actual stream iterator from the response
+ # Keep the botocore EventStream intact. Its __iter__ is a generator
+ # function, so calling iter() here would open a second generator
+ # over the one-shot event body before the caller reads
+ # response["stream"].
if isinstance(wrapped_response, dict) and "stream" in wrapped_response:
- self.__wrapped_stream = iter(wrapped_response["stream"])
+ self._event_stream = wrapped_response["stream"]
else:
- self.__wrapped_stream = iter(wrapped_response)
+ self._event_stream = wrapped_response
+ self._iterator = None
+ self._closed = False
self._span = span
self._span_name = span_name
@@ -178,17 +183,21 @@ def converse_stream(
self._server_address = server_address
self._server_port = server_port
self._event_provider = event_provider
+ self._streaming_response_processed = False
+ self._stream_error = None
def __enter__(self):
- if hasattr(self.__wrapped_stream, "__enter__"):
- self.__wrapped_stream.__enter__()
+ # The boto3 response dict is not a context manager. Callers that
+ # use ``with client.converse_stream()`` are using this wrapper.
return self
def __exit__(self, exc_type, exc_value, traceback):
- if hasattr(self.__wrapped_stream, "__exit__"):
- self.__wrapped_stream.__exit__(exc_type, exc_value, traceback)
+ # A raise from user code is a cancel. A failure raised by the
+ # event stream was stored on _stream_error in __next__.
+ self.close()
def __iter__(self):
+ self._ensure_iterator()
return self
def __getattr__(self, name):
@@ -196,25 +205,108 @@ def converse_stream(
return getattr(self.__wrapped_response, name)
def get(self, key, default=None):
- """Delegate get method to the wrapped response if its a dict."""
+ """Return this wrapper for ``stream``; delegate every other key.
+
+ boto3 callers do ``for event in response["stream"]``. The raw
+ EventStream under that key never enters __next__, so the span
+ would stay open and the chunks would never be recorded.
+ """
+ if key == "stream":
+ return self
if isinstance(self.__wrapped_response, dict):
return self.__wrapped_response.get(key, default)
return getattr(self.__wrapped_response, key, default)
def __getitem__(self, key):
- """Delegate item access to the wrapped response if its a dict."""
+ """Return this wrapper for ``stream``; delegate every other key."""
+ if key == "stream":
+ return self
if isinstance(self.__wrapped_response, dict):
return self.__wrapped_response[key]
return getattr(self.__wrapped_response, key)
+ def close(self):
+ """Release the HTTP body and end the span once.
+
+ ``EventStream.__iter__`` returns a generator. That generator's
+ close() does not call ``EventStream.close()``, and
+ ``StreamingBody.__del__`` does not close the socket.
+ """
+ try:
+ self._close_body()
+ finally:
+ self._finalize_streaming_span(ended_cleanly=False)
+
+ def __del__(self):
+ """End a span abandoned by a bare ``for`` loop.
+
+ ``break`` does not call close() on this iterator. Nothing else
+ closes the botocore stream, so dropping the response has to.
+ """
+ try:
+ self.close()
+ except Exception:
+ pass
+
+ def _ensure_iterator(self):
+ if self._iterator is None:
+ self._iterator = iter(self._event_stream)
+ return self._iterator
+
+ def _close_body(self):
+ if self._closed:
+ return
+ self._closed = True
+ iterator = self._iterator
+ event_stream = self._event_stream
+ # The generator and the EventStream are different objects.
+ # Closing only the generator leaves the HTTP connection open.
+ if iterator is not None and iterator is not event_stream:
+ iterator_close = getattr(iterator, "close", None)
+ if iterator_close is not None:
+ try:
+ iterator_close()
+ except Exception:
+ pass
+ stream_close = getattr(event_stream, "close", None)
+ if stream_close is not None:
+ try:
+ stream_close()
+ except Exception:
+ pass
+
def __next__(self):
try:
- chunk = next(self.__wrapped_stream)
- process_chunk(self, chunk)
- return chunk
+ chunk = next(self._ensure_iterator())
except StopIteration:
- try:
- with self._span:
+ self._finalize_streaming_span(ended_cleanly=True)
+ self._close_body()
+ raise
+ except Exception as exc:
+ self._stream_error = exc
+ raise
+ process_chunk(self, chunk)
+ return chunk
+
+ def _finalize_streaming_span(self, ended_cleanly):
+ """End the span once.
+
+ ``ended_cleanly`` is True only for StopIteration. An empty finish
+ reason is otherwise a cancel. A failure raised by the event stream
+ is recorded after attributes are set, so partial text stays on the
+ span and the status is ERROR rather than the OK that
+ ``process_streaming_chat_response`` writes when nothing failed.
+ """
+ if self._streaming_response_processed:
+ return
+ self._streaming_response_processed = True
+ if self._stream_error is not None and not self._finish_reason:
+ self._finish_reason = "error"
+ elif not ended_cleanly and not self._finish_reason:
+ self._finish_reason = "incomplete"
+ try:
+ with self._span:
+ try:
process_streaming_chat_response(
self,
pricing_info=pricing_info,
@@ -226,11 +318,11 @@ def converse_stream(
version=version,
event_provider=self._event_provider,
)
-
- except Exception as e:
- handle_exception(self._span, e)
-
- raise
+ finally:
+ if self._stream_error is not None:
+ handle_exception(self._span, self._stream_error)
+ except Exception as e:
+ handle_exception(self._span, e)
def wrapper(wrapped, instance, args, kwargs):
"""
diff --git a/sdk/python/src/openlit/instrumentation/bedrock/utils.py b/sdk/python/src/openlit/instrumentation/bedrock/utils.py
index 26642143..c7619c41 100644
--- a/sdk/python/src/openlit/instrumentation/bedrock/utils.py
+++ b/sdk/python/src/openlit/instrumentation/bedrock/utils.py
@@ -602,7 +602,12 @@ def common_chat_logic(
except Exception as e:
logger.warning("Failed to emit inference event: %s", e, exc_info=True)
- scope._span.set_status(Status(StatusCode.OK))
+ # OK is sticky in the OTel SDK: a later set_status(ERROR) is ignored.
+ # Stream failures set scope._stream_error before this runs, then
+ # handle_exception records ERROR once these attributes are written.
+ # Non-streaming scopes never set _stream_error, so they still record OK.
+ if getattr(scope, "_stream_error", None) is None:
+ scope._span.set_status(Status(StatusCode.OK))
# Record metrics
if not disable_metrics:
diff --git a/sdk/python/tests/test_bedrock_stream_span_lifecycle.py b/sdk/python/tests/test_bedrock_stream_span_lifecycle.py
new file mode 100644
index 00000000..e33abab0
--- /dev/null
+++ b/sdk/python/tests/test_bedrock_stream_span_lifecycle.py
@@ -0,0 +1,325 @@
+# pylint: disable=protected-access, too-few-public-methods, not-context-manager
+"""Bedrock converse_stream spans must export on the path boto3 callers use.
+
+``converse_stream`` returns a dict. Production code iterates
+``response["stream"]`` and may call ``close()`` on that object. botocore's
+``EventStream.__iter__`` is a generator function: ``iter(stream)`` is not the
+object ``close()`` releases the HTTP body on, and a bare ``for`` plus ``break``
+never calls ``close()``.
+
+The fake stream below has that shape. A self-iterating iterator would hide
+both the missed span and the leaked connection.
+"""
+
+import gc
+
+import pytest
+from opentelemetry import context as context_api
+from opentelemetry import trace as trace_api
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+ InMemorySpanExporter,
+)
+from opentelemetry.trace import StatusCode
+
+from openlit._config import OpenlitConfig
+from openlit.instrumentation.bedrock.bedrock import converse_stream
+from openlit.semcov import SemanticConvention
+
+REQUEST_KWARGS = {"modelId": "amazon.titan-text-express-v1"}
+
+
+def _delta(text):
+ return {
+ "contentBlockDelta": {
+ "delta": {"text": text},
+ "contentBlockIndex": 0,
+ }
+ }
+
+
+def _terminal_events():
+ return [
+ {"messageStop": {"stopReason": "end_turn"}},
+ {
+ "metadata": {
+ "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15},
+ "metrics": {"latencyMs": 42},
+ }
+ },
+ ]
+
+
+class FakeEventStream:
+ """botocore EventStream: ``__iter__`` yields, ``close()`` closes the body."""
+
+ def __init__(self, events, raise_after=None):
+ self._events = list(events)
+ self._raise_after = raise_after
+ self.closed = False
+ self.iter_calls = 0
+
+ def __iter__(self):
+ self.iter_calls += 1
+ for index, event in enumerate(self._events):
+ if self.closed:
+ return
+ if self._raise_after is not None and index >= self._raise_after:
+ raise RuntimeError("stream failed")
+ yield event
+
+ def close(self):
+ """Record the HTTP-body close. Generator.close() must not be enough."""
+ self.closed = True
+
+
+class FakeClient:
+ """Minimal stand-in for a bedrock-runtime client."""
+
+ def __init__(self, stream):
+ self.stream = stream
+ self.calls = []
+
+ def converse_stream(self, **kwargs):
+ """Return a boto3-shaped streaming response."""
+ self.calls.append(kwargs)
+ return {"stream": self.stream, "ResponseMetadata": {"RequestId": "req-1"}}
+
+
+def _tracer_and_exporter():
+ OpenlitConfig.reset_to_defaults()
+ exporter = InMemorySpanExporter()
+ provider = TracerProvider()
+ provider.add_span_processor(SimpleSpanProcessor(exporter))
+ return provider.get_tracer(__name__), exporter
+
+
+def _client(tracer, events, raise_after=None, capture_message_content=True):
+ wrapper = converse_stream(
+ version="test",
+ environment="test",
+ application_name="test",
+ tracer=tracer,
+ pricing_info={},
+ capture_message_content=capture_message_content,
+ metrics=None,
+ disable_metrics=True,
+ )
+ fake = FakeEventStream(events, raise_after=raise_after)
+ raw = FakeClient(fake)
+ client = wrapper(
+ lambda *args, **kwargs: raw,
+ None,
+ (),
+ {"service_name": "bedrock-runtime"},
+ )
+ return client, fake
+
+
+def _finish_reasons(span):
+ return tuple(span.attributes[SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON])
+
+
+def _output(span):
+ return span.attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
+
+
+def _assert_tokens(span):
+ assert span.attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 10
+ assert span.attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 5
+
+
+def test_constructing_the_wrapper_does_not_read_or_close_the_stream():
+ """``iter()`` in ``__init__`` would open a second generator over the body."""
+
+ tracer, _exporter = _tracer_and_exporter()
+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])
+
+ response = client.converse_stream(**REQUEST_KWARGS)
+ stream = response.get("stream")
+
+ assert stream is response
+ assert response["ResponseMetadata"]["RequestId"] == "req-1"
+ assert fake.iter_calls == 0
+ assert fake.closed is False
+
+
+def test_break_on_response_stream_ends_span_and_closes_body():
+ """The AWS sample loop must export one partial span and close the body."""
+
+ tracer, exporter = _tracer_and_exporter()
+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])
+
+ response = client.converse_stream(**REQUEST_KWARGS)
+ for event in response["stream"]:
+ assert "partial" in event["contentBlockDelta"]["delta"]["text"]
+ break
+
+ del response
+ gc.collect()
+
+ assert fake.closed
+ assert fake.iter_calls == 1
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].status.status_code == StatusCode.OK
+ assert _finish_reasons(spans[0]) == ("incomplete",)
+ assert "partial" in _output(spans[0])
+ assert "answer" not in _output(spans[0])
+
+
+def test_get_stream_break_inside_with_ends_span():
+ """``with`` plus ``response.get('stream')`` is the same iterator and span."""
+
+ tracer, exporter = _tracer_and_exporter()
+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])
+
+ with client.converse_stream(**REQUEST_KWARGS) as response:
+ stream = response.get("stream")
+ assert stream is response
+ for _event in stream:
+ break
+
+ assert fake.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].status.status_code == StatusCode.OK
+ assert _finish_reasons(spans[0]) == ("incomplete",)
+ assert "partial" in _output(spans[0])
+ assert "answer" not in _output(spans[0])
+
+
+def test_held_stream_stays_open_until_released():
+ """Keeping the response must leave the next event readable."""
+
+ tracer, exporter = _tracer_and_exporter()
+ client, fake = _client(
+ tracer, [_delta("one"), _delta("two"), _delta("three")]
+ )
+
+ response = client.converse_stream(**REQUEST_KWARGS)
+ stream = response["stream"]
+ for event in stream:
+ assert event["contentBlockDelta"]["delta"]["text"] == "one"
+ break
+
+ assert not exporter.get_finished_spans()
+ assert fake.closed is False
+ assert next(stream)["contentBlockDelta"]["delta"]["text"] == "two"
+
+ del response
+ del stream
+ gc.collect()
+
+ assert fake.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert _finish_reasons(spans[0]) == ("incomplete",)
+ output = _output(spans[0])
+ assert "one" in output
+ assert "two" in output
+ assert "three" not in output
+
+
+def test_full_consumption_of_response_stream_exports_exactly_one_span():
+ """StopIteration then close() must not emit a second span or rewrite the reason."""
+
+ tracer, exporter = _tracer_and_exporter()
+ events = [_delta("partial"), _delta(" answer"), *_terminal_events()]
+ client, fake = _client(tracer, events)
+
+ response = client.converse_stream(**REQUEST_KWARGS)
+ for _event in response["stream"]:
+ pass
+ for _event in response["stream"]:
+ pass
+ response["stream"].close()
+
+ assert fake.closed
+ assert fake.iter_calls == 1
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].status.status_code == StatusCode.OK
+ assert _finish_reasons(spans[0]) == ("end_turn",)
+ assert "partial answer" in _output(spans[0])
+ _assert_tokens(spans[0])
+
+
+def test_close_on_response_stream_finalizes_once():
+ """``response['stream'].close()`` ends the span and the HTTP body."""
+
+ tracer, exporter = _tracer_and_exporter()
+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])
+
+ response = client.converse_stream(**REQUEST_KWARGS)
+ next(response["stream"])
+ response["stream"].close()
+ response.close()
+
+ assert fake.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].status.status_code == StatusCode.OK
+ assert _finish_reasons(spans[0]) == ("incomplete",)
+ assert "partial" in _output(spans[0])
+ assert "answer" not in _output(spans[0])
+
+
+def test_caller_exception_inside_with_stays_ok():
+ """A raise from user code is a cancel, not a Bedrock failure."""
+
+ tracer, exporter = _tracer_and_exporter()
+ client, fake = _client(tracer, [_delta("partial"), _delta("answer")])
+
+ with pytest.raises(ValueError, match="caller"):
+ with client.converse_stream(**REQUEST_KWARGS) as response:
+ for _event in response["stream"]:
+ raise ValueError("caller")
+
+ assert fake.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].status.status_code == StatusCode.OK
+ assert _finish_reasons(spans[0]) == ("incomplete",)
+ assert SemanticConvention.ERROR_TYPE not in spans[0].attributes
+ assert all(event.name != "exception" for event in spans[0].events)
+ assert "partial" in _output(spans[0])
+ assert "answer" not in _output(spans[0])
+
+
+def test_stream_error_inside_with_marks_span_error():
+ """A failure from the event stream is a failed span, with text already seen."""
+
+ tracer, exporter = _tracer_and_exporter()
+ client, fake = _client(
+ tracer, [_delta("partial"), _delta("answer")], raise_after=1
+ )
+
+ with pytest.raises(RuntimeError, match="stream failed"):
+ with client.converse_stream(**REQUEST_KWARGS) as response:
+ for _event in response["stream"]:
+ pass
+
+ assert fake.closed
+ spans = exporter.get_finished_spans()
+ assert len(spans) == 1
+ assert spans[0].status.status_code == StatusCode.ERROR
+ assert spans[0].attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"
+ assert _finish_reasons(spans[0]) == ("error",)
+ assert "partial" in _output(spans[0])
+ assert "answer" not in _output(spans[0])
+
+
+def test_context_detached_after_early_break():
+ """Finalizing the span must not leave it attached to the OTel context."""
+
+ tracer, _exporter = _tracer_and_exporter()
+ client, _fake = _client(tracer, [_delta("partial")])
+
+ with client.converse_stream(**REQUEST_KWARGS) as response:
+ next(response["stream"])
+
+ assert (
+ trace_api.get_current_span(context_api.get_current()) is trace_api.INVALID_SPAN
+ )