~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Tests for LLM rate limiting functionality.
"""
import pytest
from unittest.mock import AsyncMock, Mock, patch
from langchain_core.messages import AIMessage
from tests.test_utils import add_src_to_path
# Add src to path
add_src_to_path()
from local_deep_research.web_search_engines.rate_limiting.llm import ( # noqa: E402
create_rate_limited_llm_wrapper,
is_llm_rate_limit_error,
)
class TestLLMRateLimitDetection:
"""Test rate limit error detection for various LLM providers."""
def test_detects_429_status_code(self):
"""Test detection of HTTP 429 status code."""
error = Mock()
error.response = Mock()
error.response.status_code = 429
assert is_llm_rate_limit_error(error) is True
def test_detects_rate_limit_messages(self):
"""Test detection of common rate limit error messages."""
test_messages = [
"Error: 429 Resource has been exhausted (e.g. check quota).",
"Rate limit exceeded. Please try again later.",
"Too many requests. Quota exceeded.",
"API rate limit: maximum requests per minute exceeded",
"Threshold for requests has been reached",
]
for message in test_messages:
error = Exception(message)
assert is_llm_rate_limit_error(error) is True, (
f"Failed to detect: {message}"
)
def test_does_not_detect_non_rate_limit_errors(self):
"""Test that non-rate limit errors are not detected."""
test_messages = [
"Model not found",
"Invalid API key",
"Connection refused",
"Internal server error",
]
for message in test_messages:
error = Exception(message)
assert is_llm_rate_limit_error(error) is False, (
f"Incorrectly detected: {message}"
)
class TestRateLimitedLLMWrapper:
"""Test the rate limited LLM wrapper functionality."""
@pytest.fixture
def mock_llm(self):
"""Create a mock LLM for testing."""
llm = Mock()
llm.model_name = "test-model"
llm.base_url = "https://api.example.com"
llm.invoke = Mock(return_value=AIMessage(content="Test response"))
return llm
@pytest.fixture
def mock_db_settings(self):
"""Mock database settings."""
# Rate limiting is now disabled by default in the wrapper
# This fixture is no longer needed but kept for compatibility
yield None
def test_wrapper_creation_with_rate_limiting_disabled_by_default(
self, mock_llm, mock_db_settings
):
"""Test wrapper creation - rate limiting is now disabled by default."""
wrapper = create_rate_limited_llm_wrapper(mock_llm, provider="openai")
assert wrapper is not None
assert hasattr(wrapper, "invoke")
assert wrapper.base_llm == mock_llm
assert wrapper.provider == "openai"
# Rate limiting is disabled by default now
assert wrapper.rate_limiter is None
def test_wrapper_creation_always_disabled(self, mock_llm):
"""Test wrapper creation - rate limiting is always disabled now."""
# No need to patch, rate limiting is disabled by default
wrapper = create_rate_limited_llm_wrapper(mock_llm, provider="openai")
assert wrapper.rate_limiter is None
def test_local_providers_skip_rate_limiting(
self, mock_llm, mock_db_settings
):
"""Test that local providers skip rate limiting even when enabled."""
local_providers = ["ollama", "lmstudio", "llamacpp"]
for provider in local_providers:
wrapper = create_rate_limited_llm_wrapper(
mock_llm, provider=provider
)
assert wrapper.rate_limiter is None, (
f"Rate limiting should be skipped for {provider}"
)
def test_rate_limit_key_generation(self, mock_llm, mock_db_settings):
"""Test the generation of rate limit keys."""
wrapper = create_rate_limited_llm_wrapper(mock_llm, provider="openai")
key = wrapper._get_rate_limit_key()
assert key == "openai-api.example.com-test-model"
def test_invoke_without_rate_limiting(self, mock_llm):
"""Test invoke when rate limiting is disabled (always the case now)."""
# No need to patch, rate limiting is disabled by default
wrapper = create_rate_limited_llm_wrapper(mock_llm, provider="openai")
response = wrapper.invoke("Test prompt")
assert response.content == "Test response"
mock_llm.invoke.assert_called_once_with("Test prompt")
def test_attribute_passthrough(self, mock_llm, mock_db_settings):
"""Test that attributes are passed through to the base LLM."""
mock_llm.custom_attribute = "test_value"
wrapper = create_rate_limited_llm_wrapper(mock_llm, provider="openai")
assert wrapper.custom_attribute == "test_value"
assert wrapper.model_name == "test-model"
class TestIntegrationWithTracker:
"""Test integration with the adaptive rate limit tracker."""
@patch(
"local_deep_research.web_search_engines.rate_limiting.llm.wrapper.get_tracker"
)
def test_tracker_not_used_when_disabled(self, mock_get_tracker):
"""Test that tracker is not used since rate limiting is disabled."""
# Rate limiting is disabled by default, so tracker shouldn't be called
mock_tracker = Mock()
mock_tracker.get_wait_time.return_value = 0
mock_get_tracker.return_value = mock_tracker
mock_llm = Mock()
mock_llm.model_name = "test-model"
mock_llm.base_url = "https://api.example.com"
mock_llm.invoke.return_value = AIMessage(content="Success")
wrapper = create_rate_limited_llm_wrapper(mock_llm, provider="openai")
# Verify rate limiter is None (disabled)
assert wrapper.rate_limiter is None
# get_tracker should not be called since rate limiting is disabled
mock_get_tracker.assert_not_called()
# But invoke should still work
result = wrapper.invoke("Test prompt")
assert result.content == "Success"
if __name__ == "__main__":
pytest.main([__file__, "-v"])
class TestRateLimitedWrapperLangChainSurface:
"""Issue #6296: every public LangChain entry point must be handled by the
wrapper itself — not resolved through __getattr__ to the raw model, which
skips scrubbing and rate-limit bookkeeping."""
LANGCHAIN_ENTRY_POINTS = [
"invoke",
"ainvoke",
"stream",
"astream",
"batch",
"abatch",
"generate",
"agenerate",
"bind_tools",
"with_structured_output",
]
@pytest.fixture
def mock_llm(self):
llm = Mock()
llm.model_name = "test-model"
llm.base_url = "https://api.example.com"
llm.invoke = Mock(return_value=AIMessage(content="ok"))
return llm
def _wrapper(self, mock_llm):
return create_rate_limited_llm_wrapper(mock_llm, provider="openai")
def test_rate_limit_error_is_scrubbed_on_stream(self, mock_llm):
"""A 429 raised when the stream starts must not leak the body."""
fake_key = "sk-" + "a1b2c3d4e5f6g7h8i9j0" # 20+ chars: real key shape
err = Exception(f"Error: 429 quota exceeded, key {fake_key}")
mock_llm.stream = Mock(side_effect=err)
wrapper = self._wrapper(mock_llm)
with pytest.raises(Exception) as exc_info:
list(wrapper.stream("prompt"))
assert fake_key not in str(exc_info.value)
def test_stream_forwards_chunks(self, mock_llm):
mock_llm.stream = Mock(return_value=iter(["a", "b"]))
wrapper = self._wrapper(mock_llm)
chunks = list(wrapper.stream("prompt"))
assert chunks == ["a", "b"]
mock_llm.stream.assert_called_once_with("prompt")
# `generate`/`agenerate` are declared above as surfaces that should not
# bypass the wrapper, but neither is overridden yet: both still resolve
# through __getattr__ to the raw model. Pinned rather than described, so
# closing the gap reds this cell instead of leaving the list stale.
ENTRY_POINTS_STILL_FORWARDED = ["generate", "agenerate"]
def test_declared_entry_points_are_defined_or_pinned_as_forwarded(
self, mock_llm
):
"""The entry-point list documented the contract and nothing asserted
it, which is how a method named `_astream` — defined but never
resolvable, since __getattr__ answers `astream` from the raw model —
passed review with a green test (#6447)."""
wrapper = self._wrapper(mock_llm)
forwarded = [
name
for name in self.LANGCHAIN_ENTRY_POINTS
if name not in type(wrapper).__dict__
]
assert forwarded == self.ENTRY_POINTS_STILL_FORWARDED
def test_astream_is_defined_on_the_wrapper(self, mock_llm):
"""Regression for the underscore: an `_astream` twin looks like
coverage and is never reached."""
wrapper = self._wrapper(mock_llm)
assert "astream" in type(wrapper).__dict__
def test_rate_limit_error_during_stream_iteration_is_scrubbed(
self, mock_llm
):
"""`BaseChatModel.stream` is a generator function: calling it returns a
generator without making the request, so a 429 arrives on the first
`next()`. A scrub wrapped around the call alone never sees it."""
fake_key = "sk-" + "a1b2c3d4e5f6g7h8i9j0"
err = Exception(f"Error: 429 quota exceeded, key {fake_key}")
def failing_stream(*args, **kwargs):
yield "partial"
raise err
mock_llm.stream = failing_stream
wrapper = self._wrapper(mock_llm)
with pytest.raises(Exception) as exc_info:
list(wrapper.stream("prompt"))
assert fake_key not in str(exc_info.value)
assert "429" not in str(exc_info.value.__cause__ or "")
def test_stream_yields_chunks_before_the_failure(self, mock_llm):
"""The scrub must not turn the stream into a buffer: whatever arrived
before the 429 still reaches the caller."""
def failing_stream(*args, **kwargs):
yield "first"
raise Exception("Error: 429 quota exceeded")
mock_llm.stream = failing_stream
wrapper = self._wrapper(mock_llm)
seen = []
with pytest.raises(Exception):
for chunk in wrapper.stream("prompt"):
seen.append(chunk)
assert seen == ["first"]
def test_consumer_thrown_errors_are_not_scrubbed(self, mock_llm):
"""The scrub belongs to the provider side of the stream. A handler
placed around the `yield` would also catch what the CONSUMER throws
back in, so a caller-side failure that happens to look like a 429
would come back as this wrapper's RateLimitError and point the blame
at the provider."""
def ok_stream(*args, **kwargs):
yield "first"
yield "second"
mock_llm.stream = ok_stream
wrapper = self._wrapper(mock_llm)
stream = wrapper.stream("prompt")
assert next(stream) == "first"
caller_side = ValueError("Error: 429 quota exceeded, from my own loop")
with pytest.raises(ValueError) as exc_info:
stream.throw(caller_side)
assert exc_info.value is caller_side
def test_rate_limit_error_during_astream_iteration_is_scrubbed(
self, mock_llm
):
"""Same shape on the async side: `astream` is an async generator
function, so the provider error arrives on the first `__anext__()`."""
import asyncio
fake_key = "sk-" + "z9y8x7w6v5u4t3s2r1q0"
async def failing_astream(*args, **kwargs):
yield "partial"
raise Exception(f"Error: 429 quota exceeded, key {fake_key}")
mock_llm.astream = failing_astream
wrapper = self._wrapper(mock_llm)
async def consume():
return [chunk async for chunk in wrapper.astream("prompt")]
with pytest.raises(Exception) as exc_info:
asyncio.run(consume())
assert fake_key not in str(exc_info.value)
def test_bind_tools_rewraps_bound_model(self, mock_llm):
"""bind_tools must return a wrapper, not the raw bound model."""
bound = Mock()
mock_llm.bind_tools = Mock(return_value=bound)
wrapper = self._wrapper(mock_llm)
result = wrapper.bind_tools([])
mock_llm.bind_tools.assert_called_once_with([])
assert result.base_llm is bound
def test_with_structured_output_rewraps(self, mock_llm):
bound = Mock()
mock_llm.with_structured_output = Mock(return_value=bound)
wrapper = self._wrapper(mock_llm)
result = wrapper.with_structured_output({"type": "object"})
mock_llm.with_structured_output.assert_called_once_with(
{"type": "object"}
)
assert result.base_llm is bound
def test_batch_preserves_order_and_results(self, mock_llm):
wrapper = self._wrapper(mock_llm)
calls = []
mock_llm.invoke = Mock(
side_effect=lambda m, cfg=None, **kw: (
calls.append(m) or AIMessage(content=f"r-{m}")
)
)
results = wrapper.batch(["m1", "m2"])
assert [r.content for r in results] == ["r-m1", "r-m2"]
def test_batch_return_exceptions_collects_failures(self, mock_llm):
wrapper = self._wrapper(mock_llm)
mock_llm.invoke = Mock(
side_effect=[AIMessage(content="ok"), Exception("boom")]
)
results = wrapper.batch(["m1", "m2"], return_exceptions=True)
assert results[0].content == "ok"
assert isinstance(results[1], Exception)
def test_batch_raises_without_return_exceptions(self, mock_llm):
wrapper = self._wrapper(mock_llm)
mock_llm.invoke = Mock(side_effect=Exception("boom"))
with pytest.raises(Exception, match="boom"):
wrapper.batch(["m1"])
def test_rate_limit_error_wrapped_in_batch(self, mock_llm):
"""Batch through invoke() inherits the 429 scrub: the provider body
must not surface raw."""
wrapper = self._wrapper(mock_llm)
fake_key = "sk-" + "z9y8x7w6v5u4t3s2r1q0"
mock_llm.invoke = Mock(
side_effect=Exception(f"429 quota, auth Bearer {fake_key}")
)
results = wrapper.batch(["m1"], return_exceptions=True)
assert fake_key not in str(results[0])
def test_scrub_applies_even_with_rate_limiter_disabled(self, mock_llm):
"""Rate limiting is off by default; the credential scrub must not be
gated on it."""
wrapper = self._wrapper(mock_llm)
fake_key = "sk-" + "m1n2o3p4q5r6s7t8u9v0"
mock_llm.invoke = Mock(
side_effect=Exception(f"429 quota, auth Bearer {fake_key}")
)
with pytest.raises(Exception) as exc_info:
wrapper.invoke("prompt")
assert fake_key not in str(exc_info.value)
class TestRateLimitedWrapperReviewFixes:
"""Regression coverage for the maintainer CHANGES_REQUESTED on #6347."""
@pytest.fixture
def mock_llm(self):
llm = Mock()
llm.model_name = "test-model"
llm.base_url = "https://api.example.com"
llm.invoke = Mock(return_value=AIMessage(content="ok"))
return llm
def _wrapper(self, mock_llm):
return create_rate_limited_llm_wrapper(mock_llm, provider="openai")
def test_astream_is_async_iterable(self, mock_llm):
"""`async for chunk in wrapper.astream(...)` must work without
awaiting astream first (LangChain contract)."""
import asyncio
async def fake_astream(*args, **kwargs):
for chunk in ["c1", "c2"]:
yield chunk
mock_llm.astream = fake_astream
wrapper = self._wrapper(mock_llm)
async def consume():
return [chunk async for chunk in wrapper.astream("prompt")]
chunks = asyncio.run(consume())
assert chunks == ["c1", "c2"]
def test_batch_forwards_shared_config(self, mock_llm):
"""A shared config must reach the underlying invoke."""
wrapper = self._wrapper(mock_llm)
captured = []
mock_llm.invoke = Mock(
side_effect=lambda msg, cfg=None, **kw: (
captured.append(cfg) or AIMessage(content="ok")
)
)
wrapper.batch(["m1", "m2"], {"tags": ["t1"]})
assert captured == [{"tags": ["t1"]}, {"tags": ["t1"]}]
def test_batch_forwards_per_input_configs(self, mock_llm):
wrapper = self._wrapper(mock_llm)
captured = []
mock_llm.invoke = Mock(
side_effect=lambda msg, cfg=None, **kw: (
captured.append(cfg) or AIMessage(content="ok")
)
)
per_input = [{"metadata": 1}, {"metadata": 2}]
wrapper.batch(["m1", "m2"], per_input)
assert captured == per_input
def test_batch_config_mismatch_raises(self, mock_llm):
wrapper = self._wrapper(mock_llm)
with pytest.raises(ValueError, match="configs must match"):
wrapper.batch(["m1", "m2"], [{"only": "one"}])
def test_abatch_forwards_shared_and_per_input_configs(self, mock_llm):
"""Async batches must forward shared and per-input configs too
(review parity for the sync batch cases)."""
import asyncio
wrapper = self._wrapper(mock_llm)
captured = []
async def fake_ainvoke(msg, cfg=None, **kw):
captured.append(cfg)
return AIMessage(content=f"r-{msg}")
mock_llm.ainvoke = fake_ainvoke
async def run():
await wrapper.abatch(["m1"], {"tags": ["shared"]})
await wrapper.abatch(
["m1", "m2"], [{"metadata": 1}, {"metadata": 2}]
)
asyncio.run(run())
assert captured[:1] == [{"tags": ["shared"]}]
assert captured[1:] == [{"metadata": 1}, {"metadata": 2}]
class TestStreamUpstreamLifetime:
"""Upstream generator lifetime in stream()/astream() (#6607).
The `yield` stays outside the scrub region so a consumer `throw()` is
not scrubbed as a provider error; the wrapper must nonetheless close
the upstream generator on early consumer close instead of leaking the
connection until GC.
Every test holds each upstream generator the wrapper creates in an
outer list: otherwise CPython refcounting closes the upstream as soon
as the wrapper's frame is freed, and the assertions would pass with
the explicit close removed.
"""
FAKE_KEY = "sk-" + "a1b2c3d4e5f6g7h8i9j0" # 20+ chars: real key shape
@pytest.fixture
def mock_llm(self):
llm = Mock()
llm.model_name = "test-model"
llm.base_url = "https://api.example.com"
return llm
def _wrapper(self, mock_llm):
return create_rate_limited_llm_wrapper(mock_llm, provider="openai")
@staticmethod
def _tracked(factory, upstream):
def tracked(*args, **kwargs):
gen = factory(*args, **kwargs)
upstream.append(gen)
return gen
return tracked
def test_stream_closes_upstream_on_consumer_close(self, mock_llm):
events: list[str] = []
upstream: list = []
def source(*args, **kwargs):
try:
for chunk in ["c1", "c2", "c3"]:
events.append(f"yield:{chunk}")
yield chunk
finally:
events.append("upstream_closed")
mock_llm.stream = self._tracked(source, upstream)
gen = self._wrapper(mock_llm).stream("prompt")
assert next(gen) == "c1"
gen.close()
assert len(upstream) == 1
assert events == ["yield:c1", "upstream_closed"]
def test_astream_closes_upstream_on_consumer_close(self, mock_llm):
import asyncio
events: list[str] = []
# Hold every upstream generator the wrapper creates so GC/loop
# shutdown cannot close it behind our back — the WRAPPER must be
# the one issuing the aclose.
upstream: list = []
async def source(*args, **kwargs):
try:
for chunk in ["c1", "c2", "c3"]:
events.append(f"yield:{chunk}")
yield chunk
finally:
events.append("upstream_closed")
mock_llm.astream = self._tracked(source, upstream)
async def run():
gen = self._wrapper(mock_llm).astream("prompt")
assert await gen.__anext__() == "c1"
await gen.aclose()
# Snapshot INSIDE the loop: loop shutdown finalizes any
# leftover async generators, which would mask a leak.
return list(events)
assert asyncio.run(run()) == ["yield:c1", "upstream_closed"]
def test_consumer_throw_is_not_scrubbed_and_still_closes_upstream(
self, mock_llm
):
"""A consumer-thrown error must propagate as itself (not be scrubbed
as a provider failure — hence the 429-shaped message), and the
upstream generator must still be closed on the way out."""
events: list[str] = []
upstream: list = []
def source(*args, **kwargs):
try:
yield "c1"
yield "c2"
finally:
events.append("upstream_closed")
mock_llm.stream = self._tracked(source, upstream)
gen = self._wrapper(mock_llm).stream("prompt")
assert next(gen) == "c1"
thrown = ValueError("Error: 429 quota exceeded, from my own loop")
with pytest.raises(ValueError) as exc_info:
gen.throw(thrown)
assert exc_info.value is thrown
assert len(upstream) == 1
assert events == ["upstream_closed"]
def test_async_consumer_throw_is_not_scrubbed_and_still_closes_upstream(
self, mock_llm
):
"""Async twin of
test_consumer_throw_is_not_scrubbed_and_still_closes_upstream: a
consumer-thrown error must propagate as itself (not be scrubbed as
a provider failure — hence the 429-shaped message), and the
upstream async generator must still be closed on the way out."""
import asyncio
events: list[str] = []
upstream: list = []
async def source(*args, **kwargs):
try:
yield "c1"
yield "c2"
finally:
events.append("upstream_closed")
mock_llm.astream = self._tracked(source, upstream)
async def run():
gen = self._wrapper(mock_llm).astream("prompt")
assert await gen.__anext__() == "c1"
thrown = ValueError("Error: 429 quota exceeded, from my own loop")
with pytest.raises(ValueError) as exc_info:
await gen.athrow(thrown)
# Checked INSIDE the loop: loop shutdown finalizes leftover
# async generators, which would mask a missing close.
assert exc_info.value is thrown
assert len(upstream) == 1
assert events == ["upstream_closed"]
asyncio.run(run())
def test_astream_cancellation_closes_upstream(self, mock_llm):
"""Cancelling the task consuming astream() must still close the
upstream. The upstream here is a plain custom async iterator (not
an async generator): __anext__ returns immediately on the first
call and suspends in a long asyncio.sleep() on the second, so the
task can be cancelled while genuinely awaiting inside the
wrapper's astream(). The sleep duration is irrelevant to wall
time — cancellation fires long before it could ever elapse — so
this is not flaky."""
import asyncio
class SleepingUpstream:
"""Not an async generator: exercises the aclose() branch of
_aclose_upstream() that isn't gated by inspect.isasyncgen()/
ag_frame, per the wrapper's own support for non-generator
async iterables (mocks, custom iterators)."""
def __init__(self):
self.calls = 0
self.aclose_called = False
def __aiter__(self):
return self
async def __anext__(self):
self.calls += 1
if self.calls == 1:
return "c1"
await asyncio.sleep(3600)
return "c2" # pragma: no cover - never reached
async def aclose(self):
self.aclose_called = True
upstream = SleepingUpstream()
mock_llm.astream = Mock(return_value=upstream)
async def run():
gen = self._wrapper(mock_llm).astream("prompt")
first_chunk_seen = asyncio.Event()
consumed: list = []
async def consume():
async for chunk in gen:
consumed.append(chunk)
first_chunk_seen.set()
task = asyncio.create_task(consume())
await first_chunk_seen.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
# Checked INSIDE the loop: loop shutdown finalizes leftover
# async generators/tasks, which would mask a missing close.
assert consumed == ["c1"]
assert upstream.aclose_called
asyncio.run(run())
def test_stream_upstream_cleanup_error_does_not_replace_consumer_throw(
self, mock_llm
):
upstream: list = []
fake_key = self.FAKE_KEY
def source(*args, **kwargs):
try:
yield "c1"
yield "c2"
finally:
raise RuntimeError(f"Error: 429 cleanup, key {fake_key}")
mock_llm.stream = self._tracked(source, upstream)
gen = self._wrapper(mock_llm).stream("prompt")
assert next(gen) == "c1"
thrown = KeyError("consumer-side failure")
with pytest.raises(KeyError) as exc_info:
gen.throw(thrown)
assert exc_info.value is thrown
assert upstream[0].gi_frame is None # upstream was still closed
def test_stream_upstream_cleanup_error_on_close_is_scrubbed(self, mock_llm):
upstream: list = []
fake_key = self.FAKE_KEY
def source(*args, **kwargs):
try:
yield "c1"
yield "c2"
finally:
raise RuntimeError(f"Error: 429 cleanup, key {fake_key}")
mock_llm.stream = self._tracked(source, upstream)
gen = self._wrapper(mock_llm).stream("prompt")
assert next(gen) == "c1"
with pytest.raises(Exception) as exc_info:
gen.close()
assert fake_key not in str(exc_info.value)
assert exc_info.value.__cause__ is None
assert exc_info.value.__suppress_context__
def test_astream_upstream_cleanup_error_does_not_replace_consumer_throw(
self, mock_llm
):
import asyncio
upstream: list = []
fake_key = self.FAKE_KEY
async def source(*args, **kwargs):
try:
yield "c1"
yield "c2"
finally:
raise RuntimeError(f"Error: 429 cleanup, key {fake_key}")
mock_llm.astream = self._tracked(source, upstream)
thrown = KeyError("consumer-side failure")
async def run():
gen = self._wrapper(mock_llm).astream("prompt")
assert await gen.__anext__() == "c1"
with pytest.raises(KeyError) as exc_info:
await gen.athrow(thrown)
# Checked INSIDE the loop: loop shutdown finalizes leftover
# async generators, which would mask a missing close.
assert upstream[0].ag_frame is None
return exc_info.value
raised = asyncio.run(run())
assert raised is thrown
def test_astream_upstream_cleanup_error_on_close_is_scrubbed(
self, mock_llm
):
import asyncio
upstream: list = []
fake_key = self.FAKE_KEY
async def source(*args, **kwargs):
try:
yield "c1"
yield "c2"
finally:
raise RuntimeError(f"Error: 429 cleanup, key {fake_key}")
mock_llm.astream = self._tracked(source, upstream)
async def run():
gen = self._wrapper(mock_llm).astream("prompt")
assert await gen.__anext__() == "c1"
with pytest.raises(Exception) as exc_info:
await gen.aclose()
return exc_info.value
raised = asyncio.run(run())
assert fake_key not in str(raised)
assert raised.__cause__ is None
assert raised.__suppress_context__
def test_astream_tolerates_non_awaitable_aclose(self, mock_llm):
"""A source whose aclose() is a plain callable (e.g. a Mock) must
not make the wrapper await a non-awaitable."""
import asyncio
class Source:
def __init__(self):
self.aclose = Mock(return_value=None)
def __aiter__(self):
return self
async def __anext__(self):
return "c1"
source = Source()
mock_llm.astream = Mock(return_value=source)
async def run():
gen = self._wrapper(mock_llm).astream("prompt")
assert await gen.__anext__() == "c1"
await gen.aclose()
asyncio.run(run())
source.aclose.assert_called_once_with()
def test_astream_skips_aclose_on_running_upstream(self, mock_llm):
"""An upstream already mid-aclose (ag_running, e.g. being finalized
by the loop at shutdown / cyclic GC) must not get a second aclose,
which raises "asynchronous generator is already running"."""
import asyncio
class Source:
ag_running = False
def __init__(self):
self.aclose = AsyncMock()
def __aiter__(self):
return self
async def __anext__(self):
return "c1"
source = Source()
mock_llm.astream = Mock(return_value=source)
async def run():
gen = self._wrapper(mock_llm).astream("prompt")
assert await gen.__anext__() == "c1"
source.ag_running = True
await gen.aclose()
asyncio.run(run())
source.aclose.assert_not_called()
def test_stream_returns_all_chunks_when_close_raises_on_exhaustion(
self, mock_llm
):
"""A close() failure after the stream body already finished
successfully must not fail an otherwise-successful stream (#6805
R2): a non-generator source makes the exhaustion path (no
GeneratorExit involved) distinct from the consumer-close tests
above."""
class It:
def __init__(self, values):
self._it = iter(values)
def __iter__(self):
return self
def __next__(self):
return next(self._it)
def close(self):
raise RuntimeError("benign close failure")
mock_llm.stream = Mock(return_value=It([1, 2]))
assert list(self._wrapper(mock_llm).stream("prompt")) == [1, 2]
def test_astream_returns_all_chunks_when_aclose_raises_on_exhaustion(
self, mock_llm
):
"""Async twin of
test_stream_returns_all_chunks_when_close_raises_on_exhaustion:
a non-async-generator source (so the ag_frame guard never
applies) whose aclose() raises after the body finished must not
fail the astream."""
import asyncio
class It:
def __init__(self, values):
self._it = iter(values)
def __aiter__(self):
return self
async def __anext__(self):
try:
return next(self._it)
except StopIteration:
raise StopAsyncIteration
async def aclose(self):
raise RuntimeError("benign close failure")
mock_llm.astream = Mock(return_value=It([1, 2]))
async def run():
return [x async for x in self._wrapper(mock_llm).astream("prompt")]
assert asyncio.run(run()) == [1, 2]
def test_stream_closes_upstream_on_normal_exhaustion(self, mock_llm):
"""The upstream is closed even when the stream runs to completion
with no consumer interruption at all — a recording (non-raising)
close makes the call itself observable, independent of whether a
close failure would have been suppressed."""
closed = []
class It:
def __init__(self, values):
self._it = iter(values)
def __iter__(self):
return self
def __next__(self):
return next(self._it)
def close(self):
closed.append(1)
mock_llm.stream = Mock(return_value=It(["c1", "c2"]))
assert list(self._wrapper(mock_llm).stream("prompt")) == ["c1", "c2"]
assert closed == [1]
def test_astream_closes_upstream_on_normal_exhaustion(self, mock_llm):
"""Async twin of test_stream_closes_upstream_on_normal_exhaustion."""
import asyncio
closed = []
class It:
def __init__(self, values):
self._it = iter(values)
def __aiter__(self):
return self
async def __anext__(self):
try:
return next(self._it)
except StopIteration:
raise StopAsyncIteration
async def aclose(self):
closed.append(1)
mock_llm.astream = Mock(return_value=It(["c1", "c2"]))
async def run():
return [x async for x in self._wrapper(mock_llm).astream("prompt")]
assert asyncio.run(run()) == ["c1", "c2"]
assert closed == [1]
def test_stream_close_lookup_error_does_not_replace_consumer_throw(
self, mock_llm
):
"""A proxy whose __getattr__ raises when the wrapper probes for
close() must not replace the consumer's thrown exception: the
getattr lookup has to live inside the same guarded try as the
close() call itself (#6805 R4)."""
class BadCloseProxy:
def __init__(self, values):
self._it = iter(values)
def __iter__(self):
return self
def __next__(self):
return next(self._it)
def __getattr__(self, name):
raise RuntimeError(f"proxy __getattr__ blew up on {name!r}")
mock_llm.stream = Mock(return_value=BadCloseProxy(["c1", "c2"]))
gen = self._wrapper(mock_llm).stream("prompt")
assert next(gen) == "c1"
thrown = KeyError("consumer-side failure")
with pytest.raises(KeyError) as exc_info:
gen.throw(thrown)
assert exc_info.value is thrown
def test_astream_close_lookup_error_does_not_replace_consumer_throw(
self, mock_llm
):
"""Async twin of
test_stream_close_lookup_error_does_not_replace_consumer_throw:
the proxy's __getattr__ blows up on the ag_running/aclose probes
too, not just close()."""
import asyncio
class BadAcloseProxy:
def __init__(self, values):
self._it = iter(values)
def __aiter__(self):
return self
async def __anext__(self):
try:
return next(self._it)
except StopIteration:
raise StopAsyncIteration
def __getattr__(self, name):
raise RuntimeError(f"proxy __getattr__ blew up on {name!r}")
mock_llm.astream = Mock(return_value=BadAcloseProxy(["c1", "c2"]))
thrown = KeyError("consumer-side failure")
async def run():
gen = self._wrapper(mock_llm).astream("prompt")
assert await gen.__anext__() == "c1"
with pytest.raises(KeyError) as exc_info:
await gen.athrow(thrown)
return exc_info.value
raised = asyncio.run(run())
assert raised is thrown
def test_astream_skips_aclose_when_upstream_already_finished(
self, mock_llm
):
"""When the upstream is a real async generator that has already
run to completion (ag_frame is None), aclose() must not be
called on it again (#6805 R5). A plain custom Source class (as
used above for the ag_running guard) never satisfies
``inspect.isasyncgen``, so this needs a Mock built with ``spec``
on a genuinely exhausted async generator instance: Mock copies
the spec object's ``__class__``, which makes
``inspect.isasyncgen()`` (an isinstance check) see it as a real
async generator while still letting the test observe whether
aclose() was called."""
import asyncio
import inspect
async def real_source():
yield "c1"
async def run():
real = real_source()
assert await real.__anext__() == "c1"
with pytest.raises(StopAsyncIteration):
await real.__anext__()
assert inspect.isasyncgen(real)
assert real.ag_frame is None
proxy = Mock(spec=real)
proxy.ag_running = False
proxy.ag_frame = None
proxy.aclose = AsyncMock()
proxy.__anext__ = AsyncMock(
side_effect=["c1", StopAsyncIteration()]
)
class Aiter:
def __aiter__(self):
return proxy
mock_llm.astream = Mock(return_value=Aiter())
gen = self._wrapper(mock_llm).astream("prompt")
assert await gen.__anext__() == "c1"
with pytest.raises(StopAsyncIteration):
await gen.__anext__()
proxy.aclose.assert_not_called()
asyncio.run(run())