Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
# pylint: disable=protected-access, too-few-public-methods
"""Vertex AI streaming spans must export on every generator exit.
``GenerativeModel.generate_content(stream=True)`` and
``ChatSession.send_message(stream=True)`` return generators. The async twins
return async generators. Neither has ``__enter__`` / ``__aenter__``. A bare
``for`` does not call ``close()`` on this wrapper, so a ``break`` exports
only when the wrapper is released.
"""
import asyncio
import gc
import pytest
from opentelemetry import context as context_api
from opentelemetry import trace as trace_api
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
InMemorySpanExporter,
)
from opentelemetry.trace import StatusCode
from openlit._config import OpenlitConfig
from openlit.instrumentation.vertexai import async_vertexai as async_mod
from openlit.instrumentation.vertexai import vertexai as sync_mod
from openlit.semcov import SemanticConvention
REQUEST_KWARGS = {"stream": True}
class FakeUsageMetadata:
"""Shaped like GenerationResponse.usage_metadata."""
def __init__(self):
self.prompt_token_count = 10
self.candidates_token_count = 5
self.thoughts_token_count = 0
self.cached_content_token_count = 0
self.cache_creation_input_tokens = 0
class FakeChunk:
"""Shaped like a Vertex streaming chunk."""
def __init__(self, text, usage=None):
self.text = text
self.usage_metadata = usage
class FakeInstance:
"""Minimal instance so extract_vertexai_details() resolves a model name."""
def __init__(self):
self._location = "us-central1"
self._model_name = (
"projects/p/locations/us-central1/publishers/google/models/gemini-1.5-flash"
)
def _chunk(text, usage=False):
return FakeChunk(text, FakeUsageMetadata() if usage else None)
def _tracer_and_exporter():
OpenlitConfig.reset_to_defaults()
exporter = InMemorySpanExporter()
provider = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
return provider.get_tracer(__name__), exporter
def _factory(tracer, *, is_async, capture_message_content=False):
mod = async_mod if is_async else sync_mod
make = mod.async_send_message if is_async else mod.send_message
return make(
version="test",
environment="test",
application_name="test",
tracer=tracer,
pricing_info={},
capture_message_content=capture_message_content,
metrics=None,
disable_metrics=True,
)
def _sync_generator(chunks, box, raise_after=None):
"""Same shape as ``for chunk in response_stream: yield``."""
def _generate():
try:
for index, chunk in enumerate(chunks):
if raise_after is not None and index >= raise_after:
raise RuntimeError("stream failed")
yield chunk
finally:
box["closed"] = True
return _generate()
def _async_generator(chunks, box, raise_after=None):
"""Same shape as the async ``for chunk in response_stream: yield``."""
async def _generate():
try:
for index, chunk in enumerate(chunks):
if raise_after is not None and index >= raise_after:
raise RuntimeError("stream failed")
yield chunk
finally:
box["closed"] = True
return _generate()
def _finish_reasons(span):
return tuple(span.attributes[SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON])
def _assert_tokens(span):
assert span.attributes[SemanticConvention.GEN_AI_USAGE_INPUT_TOKENS] == 10
assert span.attributes[SemanticConvention.GEN_AI_USAGE_OUTPUT_TOKENS] == 5
async def _wait_for_async_close(box):
"""Async generator aclose() is scheduled from GC and can take two turns."""
for _ in range(5):
if box["closed"]:
return
gc.collect()
await asyncio.sleep(0)
def test_sync_with_break_exports_partial_span():
"""An early break inside ``with`` must end the span and close the generator."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=False, capture_message_content=True)
box = {"closed": False}
def fake_create(*_args, **_kwargs):
return _sync_generator([_chunk("hi", usage=True), _chunk("there")], box)
traced_stream = wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
with traced_stream as stream:
for _item in stream:
break
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.OK
assert _finish_reasons(spans[0]) == ("incomplete",)
assert "hi" in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
assert "there" not in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
_assert_tokens(spans[0])
def test_sync_explicit_close_exports_span():
"""Calling ``.close()`` directly must finalize."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=False, capture_message_content=True)
box = {"closed": False}
def fake_create(*_args, **_kwargs):
return _sync_generator([_chunk("hi", usage=True)], box)
traced_stream = wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
next(traced_stream)
traced_stream.close()
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.OK
assert _finish_reasons(spans[0]) == ("incomplete",)
_assert_tokens(spans[0])
def test_sync_full_consumption_exports_exactly_one_span():
"""StopIteration and the later close() must not export a second span."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=False, capture_message_content=True)
box = {"closed": False}
def fake_create(*_args, **_kwargs):
return _sync_generator([_chunk("partial"), _chunk(" answer", usage=True)], box)
traced_stream = wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
with traced_stream as stream:
for _item in stream:
pass
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.OK
assert SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON not in spans[0].attributes
output = spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
assert "partial answer" in output
_assert_tokens(spans[0])
def test_sync_bare_for_break_exports_span_when_wrapper_is_released():
"""A bare ``for`` plus ``break`` ends the span when the wrapper is released."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=False, capture_message_content=True)
box = {"closed": False}
def fake_create(*_args, **_kwargs):
return _sync_generator([_chunk("hi", usage=True), _chunk("there")], box)
def run():
traced_stream = wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
for _item in traced_stream:
break
run()
gc.collect()
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.OK
assert _finish_reasons(spans[0]) == ("incomplete",)
assert "hi" in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
assert "there" not in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
def test_sync_held_stream_stays_open_until_released():
"""Keeping the stream object must leave it resumable until that name is dropped."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=False, capture_message_content=True)
box = {"closed": False}
chunks = [_chunk("one", usage=True), _chunk("two"), _chunk("three")]
def fake_create(*_args, **_kwargs):
return _sync_generator(chunks, box)
response = wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
for item in response:
assert item.text == "one"
break
assert len(exporter.get_finished_spans()) == 0
assert box["closed"] is False
assert next(response).text == "two"
del response
gc.collect()
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert _finish_reasons(spans[0]) == ("incomplete",)
output = spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
assert "onetwo" in output
assert "three" not in output
def test_sync_stream_error_inside_with_marks_span_error():
"""A failure from the generator is a failed span, with text already seen."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=False, capture_message_content=True)
box = {"closed": False}
def fake_create(*_args, **_kwargs):
return _sync_generator(
[_chunk("hi", usage=True), _chunk("nope")], box, raise_after=1
)
traced_stream = wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
with pytest.raises(RuntimeError, match="stream failed"):
with traced_stream as stream:
for _item in stream:
pass
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.ERROR
assert spans[0].attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"
assert _finish_reasons(spans[0]) == ("error",)
assert "hi" in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
_assert_tokens(spans[0])
def test_sync_caller_exception_inside_with_stays_ok():
"""A raise from user code is not a Vertex failure. The cancel is incomplete."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=False, capture_message_content=True)
box = {"closed": False}
def fake_create(*_args, **_kwargs):
return _sync_generator([_chunk("hi", usage=True), _chunk("more")], box)
traced_stream = wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
with pytest.raises(ValueError, match="caller"):
with traced_stream as stream:
for _item in stream:
raise ValueError("caller")
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.OK
assert _finish_reasons(spans[0]) == ("incomplete",)
assert "hi" in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
def test_sync_context_detached_after_early_break():
"""Finalizing the span must not leave it attached to the OTel context."""
tracer, _exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=False)
box = {"closed": False}
def fake_create(*_args, **_kwargs):
return _sync_generator([_chunk("hi")], box)
traced_stream = wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
with traced_stream as stream:
next(stream)
assert (
trace_api.get_current_span(context_api.get_current()) is trace_api.INVALID_SPAN
)
@pytest.mark.asyncio
async def test_async_with_break_exports_partial_span():
"""Async twin of the context-manager break test."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=True, capture_message_content=True)
box = {"closed": False}
async def fake_create(*_args, **_kwargs):
return _async_generator([_chunk("hi", usage=True), _chunk("there")], box)
traced_stream = await wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
async with traced_stream as stream:
async for _item in stream:
break
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.OK
assert _finish_reasons(spans[0]) == ("incomplete",)
assert "hi" in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
assert "there" not in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
_assert_tokens(spans[0])
@pytest.mark.asyncio
async def test_async_aclose_exports_span():
"""``aclose`` is the async generator cleanup method and must finalize the span."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=True, capture_message_content=True)
box = {"closed": False}
async def fake_create(*_args, **_kwargs):
return _async_generator([_chunk("hi", usage=True)], box)
traced_stream = await wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
await anext(traced_stream)
await traced_stream.aclose()
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert _finish_reasons(spans[0]) == ("incomplete",)
_assert_tokens(spans[0])
@pytest.mark.asyncio
async def test_async_full_consumption_exports_exactly_one_span():
"""Async full drain must export one OK span and leave finish reason unset."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=True, capture_message_content=True)
box = {"closed": False}
async def fake_create(*_args, **_kwargs):
return _async_generator(
[_chunk("partial"), _chunk(" answer", usage=True)], box
)
traced_stream = await wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
async with traced_stream as stream:
async for _item in stream:
pass
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.OK
assert SemanticConvention.GEN_AI_RESPONSE_FINISH_REASON not in spans[0].attributes
output = spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
assert "partial answer" in output
_assert_tokens(spans[0])
@pytest.mark.asyncio
async def test_async_bare_for_break_exports_span_when_wrapper_is_released():
"""A bare ``async for`` ends the span in ``__del__`` without awaiting ``aclose``."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=True, capture_message_content=True)
box = {"closed": False}
async def fake_create(*_args, **_kwargs):
return _async_generator([_chunk("hi", usage=True), _chunk("there")], box)
async def run():
traced_stream = await wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
async for _item in traced_stream:
break
await run()
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert _finish_reasons(spans[0]) == ("incomplete",)
assert "hi" in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
await _wait_for_async_close(box)
assert box["closed"]
@pytest.mark.asyncio
async def test_async_stream_error_inside_with_marks_span_error():
"""Async twin of the generator-failure status test."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=True, capture_message_content=True)
box = {"closed": False}
async def fake_create(*_args, **_kwargs):
return _async_generator(
[_chunk("hi", usage=True), _chunk("nope")], box, raise_after=1
)
traced_stream = await wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
with pytest.raises(RuntimeError, match="stream failed"):
async with traced_stream as stream:
async for _item in stream:
pass
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.ERROR
assert spans[0].attributes[SemanticConvention.ERROR_TYPE] == "RuntimeError"
assert _finish_reasons(spans[0]) == ("error",)
assert "hi" in spans[0].attributes[SemanticConvention.GEN_AI_OUTPUT_MESSAGES]
@pytest.mark.asyncio
async def test_async_caller_exception_inside_with_stays_ok():
"""Async twin of the caller-abort status test."""
tracer, exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=True, capture_message_content=True)
box = {"closed": False}
async def fake_create(*_args, **_kwargs):
return _async_generator([_chunk("hi", usage=True), _chunk("more")], box)
traced_stream = await wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
with pytest.raises(ValueError, match="caller"):
async with traced_stream as stream:
async for _item in stream:
raise ValueError("caller")
assert box["closed"]
spans = exporter.get_finished_spans()
assert len(spans) == 1
assert spans[0].status.status_code == StatusCode.OK
assert _finish_reasons(spans[0]) == ("incomplete",)
@pytest.mark.asyncio
async def test_async_context_detached_after_early_break():
"""Async finalization must not leave the span attached to the OTel context."""
tracer, _exporter = _tracer_and_exporter()
wrapper = _factory(tracer, is_async=True)
box = {"closed": False}
async def fake_create(*_args, **_kwargs):
return _async_generator([_chunk("hi")], box)
traced_stream = await wrapper(fake_create, FakeInstance(), (), REQUEST_KWARGS)
async with traced_stream as stream:
await anext(stream)
assert (
trace_api.get_current_span(context_api.get_current()) is trace_api.INVALID_SPAN
)