Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(python-sdk): end together streaming spans on early exit (#1558)
Chen Yufeiyang29c774d06d1e
diff --git a/sdk/python/src/openlit/instrumentation/together/async_together.py b/sdk/python/src/openlit/instrumentation/together/async_together.pyindex 9a4d7148..07b2d831 100644--- a/sdk/python/src/openlit/instrumentation/together/async_together.py+++ b/sdk/python/src/openlit/instrumentation/together/async_together.py@@ -67,13 +67,15 @@ def async_completion(self._tbt = 0self._server_address = server_addressself._server_port = server_port+ self._streaming_response_processed = False+ self._stream_error = Noneasync def __aenter__(self):- await self.__wrapped__.__aenter__()+ # Together returns an async generator, which has no __aenter__.return selfasync def __aexit__(self, exc_type, exc_value, traceback):- await self.__wrapped__.__aexit__(exc_type, exc_value, traceback)+ await self.aclose()def __aiter__(self):return self@@ -82,14 +84,48 @@ def async_completion("""Delegate attribute access to the wrapped object."""return getattr(self.__wrapped__, name)- async def __anext__(self):+ async def aclose(self):+ """Close the wrapped async generator and finalize the span if it has not ended yet."""try:- chunk = await self.__wrapped__.__anext__()- process_chunk(self, chunk)- return chunk- except StopIteration:- try:- with self._span:+ await self.__wrapped__.aclose()+ finally:+ self._finalize_streaming_span(ended_cleanly=False)++ async def close(self):+ """Alias for aclose(), kept for callers using the older name."""+ await self.aclose()++ def __del__(self):+ """Finalize a span abandoned by a bare ``async for`` loop.++ Dealloc cannot await ``aclose()``. The wrapped async generator's+ own dealloc hook still closes the HTTP response.+ """+ try:+ self._finalize_streaming_span(ended_cleanly=False)+ except Exception:+ pass++ def _finalize_streaming_span(self, ended_cleanly):+ """End the span once.++ ``ended_cleanly`` is True only for StopAsyncIteration. An empty+ finish reason is otherwise a cancel. A failure raised by the+ wrapped stream is recorded after attributes are set, so partial+ text stays on the span and the status is ERROR rather than the+ OK that ``process_streaming_chat_response`` writes when nothing+ failed.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ if self._stream_error is not None and not self._finish_reason:+ self._finish_reason = "error"+ elif not ended_cleanly and not self._finish_reason:+ self._finish_reason = "incomplete"+ try:+ with self._span:+ try:process_streaming_chat_response(self,pricing_info=pricing_info,@@ -100,10 +136,22 @@ def async_completion(disable_metrics=disable_metrics,version=version,)+ finally:+ if self._stream_error is not None:+ handle_exception(self._span, self._stream_error)+ except Exception as e:+ handle_exception(self._span, e)- except Exception as e:- handle_exception(self._span, e)-+ async def __anext__(self):+ try:+ chunk = await self.__wrapped__.__anext__()+ process_chunk(self, chunk)+ return chunk+ except StopAsyncIteration:+ self._finalize_streaming_span(ended_cleanly=True)+ raise+ except Exception as exc:+ self._stream_error = excraiseasync def wrapper(wrapped, instance, args, kwargs):diff --git a/sdk/python/src/openlit/instrumentation/together/together.py b/sdk/python/src/openlit/instrumentation/together/together.pyindex 12d59112..a80bc93a 100644--- a/sdk/python/src/openlit/instrumentation/together/together.py+++ b/sdk/python/src/openlit/instrumentation/together/together.py@@ -67,13 +67,15 @@ def completion(self._tbt = 0self._server_address = server_addressself._server_port = server_port+ self._streaming_response_processed = False+ self._stream_error = Nonedef __enter__(self):- self.__wrapped__.__enter__()+ # Together returns a generator expression, which has no __enter__.return selfdef __exit__(self, exc_type, exc_value, traceback):- self.__wrapped__.__exit__(exc_type, exc_value, traceback)+ self.close()def __iter__(self):return self@@ -82,14 +84,43 @@ def completion("""Delegate attribute access to the wrapped object."""return getattr(self.__wrapped__, name)- def __next__(self):+ def close(self):+ """Close the wrapped generator and finalize the span if it has not ended yet."""try:- chunk = self.__wrapped__.__next__()- process_chunk(self, chunk)- return chunk- except StopIteration:- try:- with self._span:+ self.__wrapped__.close()+ finally:+ self._finalize_streaming_span(ended_cleanly=False)++ def __del__(self):+ """Finalize a span abandoned by a bare ``for`` loop.++ The wrapper is the iterator, so ``break`` never calls ``close()``.+ The wrapped generator's dealloc still closes the HTTP response.+ """+ try:+ self._finalize_streaming_span(ended_cleanly=False)+ except Exception:+ pass++ def _finalize_streaming_span(self, ended_cleanly):+ """End the span once.++ ``ended_cleanly`` is True only for StopIteration. An empty finish+ reason is otherwise a cancel. A failure raised by the wrapped+ stream is recorded after attributes are set, so partial text stays+ on the span and the status is ERROR rather than the OK that+ ``process_streaming_chat_response`` writes when nothing failed.+ """+ if self._streaming_response_processed:+ return+ self._streaming_response_processed = True+ if self._stream_error is not None and not self._finish_reason:+ self._finish_reason = "error"+ elif not ended_cleanly and not self._finish_reason:+ self._finish_reason = "incomplete"+ try:+ with self._span:+ try:process_streaming_chat_response(self,pricing_info=pricing_info,@@ -100,10 +131,22 @@ def completion(disable_metrics=disable_metrics,version=version,)+ finally:+ if self._stream_error is not None:+ handle_exception(self._span, self._stream_error)+ except Exception as e:+ handle_exception(self._span, e)- except Exception as e:- handle_exception(self._span, e)-+ def __next__(self):+ try:+ chunk = self.__wrapped__.__next__()+ process_chunk(self, chunk)+ return chunk+ except StopIteration:+ self._finalize_streaming_span(ended_cleanly=True)+ raise+ except Exception as exc:+ self._stream_error = excraisedef wrapper(wrapped, instance, args, kwargs):diff --git a/sdk/python/src/openlit/instrumentation/together/utils.py b/sdk/python/src/openlit/instrumentation/together/utils.pyindex 848d55c7..24d392d4 100644--- a/sdk/python/src/openlit/instrumentation/together/utils.py+++ b/sdk/python/src/openlit/instrumentation/together/utils.py@@ -353,7 +353,11 @@ def common_chat_logic(},)- scope._span.set_status(Status(StatusCode.OK))+ # OK is sticky in the OTel SDK: a later set_status(ERROR) is ignored.+ # Stream failures set scope._stream_error before this runs, then+ # handle_exception records ERROR once these attributes are written.+ if getattr(scope, "_stream_error", None) is None:+ scope._span.set_status(Status(StatusCode.OK))# Metricsif not disable_metrics:diff --git a/sdk/python/tests/test_together_stream_span_lifecycle.py b/sdk/python/tests/test_together_stream_span_lifecycle.pynew file mode 100644index 00000000..c50b9953--- /dev/null+++ b/sdk/python/tests/test_together_stream_span_lifecycle.py@@ -0,0 +1,461 @@+"""Tests that an early-exited Together chat stream still ends and exports its span.++``ChatCompletions.create(stream=True)`` returns a generator expression, and the+async method returns an async generator. Neither has ``__enter__`` /+``__aenter__``. A bare ``for`` does not call ``close()`` on this wrapper.+"""++import asyncio+import gc++import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)+from opentelemetry.trace import StatusCode++from openlit._config import OpenlitConfig+from openlit.instrumentation.together.async_together import async_completion+from openlit.instrumentation.together.together import completion+++def _tracer_and_exporter():+ OpenlitConfig.reset_to_defaults()+ exporter = InMemorySpanExporter()+ tracer_provider = TracerProvider()+ tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))+ return tracer_provider.get_tracer("test-together-stream-lifecycle"), exporter+++def _chat_kwargs():+ return {+ "model": "meta-llama/Meta-Llama-3-8B-Instruct-Lite",+ "messages": [{"role": "user", "content": "ping"}],+ "stream": True,+ }+++def _chunk(text):+ return {"id": "chatcmpl-together", "choices": [{"delta": {"content": text}}]}+++def _stop_chunk():+ return {+ "id": "chatcmpl-together",+ "choices": [{"delta": {}}],+ "usage": {"prompt_tokens": 3, "completion_tokens": 2},+ "finish_reason": "stop",+ }+++def _sync_wrapper(tracer, capture_message_content=False):+ return completion(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=capture_message_content,+ metrics=None,+ disable_metrics=True,+ )+++def _async_wrapper(tracer, capture_message_content=False):+ return async_completion(+ version="test-version",+ environment="test-env",+ application_name="test-app",+ tracer=tracer,+ pricing_info={},+ capture_message_content=capture_message_content,+ metrics=None,+ disable_metrics=True,+ )+++def _finish_reasons(span):+ return tuple(span.attributes["gen_ai.response.finish_reasons"])+++class _ClosingSyncSource:+ """HTTP stand-in whose ``with`` exits when the generator is closed."""++ def __init__(self, chunks, box, raise_after=None):+ self._chunks = list(chunks)+ self._box = box+ self._raise_after = raise_after+ self._index = 0++ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc_value, traceback):+ self._box["closed"] = True++ def __iter__(self):+ return self++ def __next__(self):+ if self._raise_after is not None and self._index >= self._raise_after:+ raise RuntimeError("stream failed")+ if self._index >= len(self._chunks):+ raise StopIteration+ chunk = self._chunks[self._index]+ self._index += 1+ return chunk+++def _sync_generator(chunks, box, raise_after=None):+ """Same shape as Together's ``(chunk for line in response)``."""++ def _generate():+ with _ClosingSyncSource(chunks, box, raise_after) as raw:+ yield from raw++ return _generate()+++class _ClosingAsyncSource:+ """Async twin of ``_ClosingSyncSource``."""++ def __init__(self, chunks, box, raise_after=None):+ self._chunks = list(chunks)+ self._box = box+ self._raise_after = raise_after+ self._index = 0++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc_value, traceback):+ self._box["closed"] = True++ def __aiter__(self):+ return self++ async def __anext__(self):+ if self._raise_after is not None and self._index >= self._raise_after:+ raise RuntimeError("stream failed")+ if self._index >= len(self._chunks):+ raise StopAsyncIteration+ chunk = self._chunks[self._index]+ self._index += 1+ return chunk+++def _async_generator(chunks, box, raise_after=None):+ """Same shape as Together's async generator expression."""++ async def _generate():+ async with _ClosingAsyncSource(chunks, box, raise_after) as raw:+ async for chunk in raw:+ yield chunk++ return _generate()+++async def _wait_for_async_close(box):+ """Async generator aclose() is scheduled from GC and can take two turns."""+ for _ in range(5):+ if box["closed"]:+ return+ gc.collect()+ await asyncio.sleep(0)+++def test_sync_stream_closed_via_context_manager_break_still_exports_span():+ """An early break inside ``with`` must end the span and close the generator."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _sync_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ def fake_create(*_args, **_kwargs):+ return _sync_generator([_chunk("hi"), _chunk("there")], box)++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())++ with traced_stream as stream:+ for _item in stream:+ break++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("incomplete",)+ assert "hi" in spans[0].attributes["gen_ai.output.messages"]+++def test_sync_stream_explicit_close_still_exports_span():+ """Calling ``.close()`` directly must finalize."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _sync_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ def fake_create(*_args, **_kwargs):+ return _sync_generator([_chunk("hi")], box)++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())+ next(traced_stream)+ traced_stream.close()++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert _finish_reasons(spans[0]) == ("incomplete",)+++def test_sync_stream_full_consumption_exports_exactly_one_span():+ """StopIteration inside ``with`` must not double-finalize or rewrite stop."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _sync_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ def fake_create(*_args, **_kwargs):+ return _sync_generator([_chunk("hi"), _stop_chunk()], box)++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())++ with traced_stream as stream:+ for _item in stream:+ pass++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert _finish_reasons(spans[0]) == ("stop",)+++def test_sync_bare_for_break_exports_span_when_wrapper_is_released():+ """A bare ``for`` plus ``break`` ends the span when the wrapper is released."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _sync_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ def fake_create(*_args, **_kwargs):+ return _sync_generator([_chunk("hi"), _chunk("there")], box)++ def run():+ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())+ for _item in traced_stream:+ break++ run()+ gc.collect()++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("incomplete",)+ assert "hi" in spans[0].attributes["gen_ai.output.messages"]+++def test_sync_held_stream_stays_open_until_released():+ """Keeping the stream object must leave it resumable until that name is dropped."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _sync_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}+ chunks = [_chunk("one"), _chunk("two"), _chunk("three")]++ def fake_create(*_args, **_kwargs):+ return _sync_generator(chunks, box)++ response = wrapper(fake_create, object(), [], _chat_kwargs())+ for item in response:+ assert item == chunks[0]+ break++ assert len(exporter.get_finished_spans()) == 0+ assert box["closed"] is False+ assert next(response) == chunks[1]++ del response+ gc.collect()++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert _finish_reasons(spans[0]) == ("incomplete",)+ output = spans[0].attributes["gen_ai.output.messages"]+ assert "onetwo" in output+ assert "three" not in output+++def test_sync_stream_error_inside_with_marks_span_error():+ """A failure from the generator is a failed span, with text already seen."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _sync_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ def fake_create(*_args, **_kwargs):+ return _sync_generator([_chunk("hi"), _chunk("nope")], box, raise_after=1)++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())++ with pytest.raises(RuntimeError, match="stream failed"):+ with traced_stream as stream:+ for _item in stream:+ pass++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.ERROR+ assert spans[0].attributes["error.type"] == "RuntimeError"+ assert _finish_reasons(spans[0]) == ("error",)+ assert "hi" in spans[0].attributes["gen_ai.output.messages"]+++def test_sync_caller_exception_inside_with_stays_ok():+ """A raise from user code is not a Together failure. The cancel is incomplete."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _sync_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ def fake_create(*_args, **_kwargs):+ return _sync_generator([_chunk("hi"), _chunk("more")], box)++ traced_stream = wrapper(fake_create, object(), [], _chat_kwargs())++ with pytest.raises(ValueError, match="caller"):+ with traced_stream as stream:+ for _item in stream:+ raise ValueError("caller")++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("incomplete",)+++async def test_async_stream_explicit_aclose_still_exports_span():+ """``aclose`` is the async generator cleanup method and must finalize the span."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _async_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ async def fake_create(*_args, **_kwargs):+ return _async_generator([_chunk("hi")], box)++ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())+ await anext(traced_stream)+ await traced_stream.aclose()++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert _finish_reasons(spans[0]) == ("incomplete",)+++async def test_async_stream_full_consumption_exports_exactly_one_span():+ """Async full drain must export one span and keep the real stop reason."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _async_wrapper(tracer)+ box = {"closed": False}++ async def fake_create(*_args, **_kwargs):+ return _async_generator([_chunk("hi"), _stop_chunk()], box)++ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())++ async with traced_stream as stream:+ async for _item in stream:+ pass++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert _finish_reasons(spans[0]) == ("stop",)+++async def test_async_bare_for_break_exports_span_when_wrapper_is_released():+ """A bare ``async for`` ends the span in ``__del__`` without awaiting ``aclose``."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _async_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ async def fake_create(*_args, **_kwargs):+ return _async_generator([_chunk("hi"), _chunk("there")], box)++ async def run():+ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())+ async for _item in traced_stream:+ break++ await run()+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert _finish_reasons(spans[0]) == ("incomplete",)+ assert "hi" in spans[0].attributes["gen_ai.output.messages"]++ await _wait_for_async_close(box)+ assert box["closed"]+++async def test_async_stream_error_inside_with_marks_span_error():+ """Async twin of the generator-failure status test."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _async_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ async def fake_create(*_args, **_kwargs):+ return _async_generator([_chunk("hi"), _chunk("nope")], box, raise_after=1)++ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())++ with pytest.raises(RuntimeError, match="stream failed"):+ async with traced_stream as stream:+ async for _item in stream:+ pass++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.ERROR+ assert spans[0].attributes["error.type"] == "RuntimeError"+ assert _finish_reasons(spans[0]) == ("error",)+++async def test_async_caller_exception_inside_with_stays_ok():+ """Async twin of the caller-abort status test."""++ tracer, exporter = _tracer_and_exporter()+ wrapper = _async_wrapper(tracer, capture_message_content=True)+ box = {"closed": False}++ async def fake_create(*_args, **_kwargs):+ return _async_generator([_chunk("hi"), _chunk("more")], box)++ traced_stream = await wrapper(fake_create, object(), [], _chat_kwargs())++ with pytest.raises(ValueError, match="caller"):+ async with traced_stream as stream:+ async for _item in stream:+ raise ValueError("caller")++ assert box["closed"]+ spans = exporter.get_finished_spans()+ assert len(spans) == 1+ assert spans[0].status.status_code == StatusCode.OK+ assert _finish_reasons(spans[0]) == ("incomplete",)