Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Retry pre-stream provider transport failures (#1003)
Ali Khokharccf46b88cfee
diff --git a/core/anthropic/errors.py b/core/anthropic/errors.pyindex afe093d6..81615d76 100644--- a/core/anthropic/errors.py+++ b/core/anthropic/errors.py@@ -21,6 +21,10 @@ def get_user_facing_error_message(return "Provider request timed out."if isinstance(e, httpx.ConnectTimeout):return "Could not connect to provider."+ if isinstance(e, httpx.ConnectError):+ return "Could not connect to provider."+ if isinstance(e, httpx.RemoteProtocolError):+ return "Provider connection was interrupted before a response was received."if isinstance(e, TimeoutError):if read_timeout_s is not None:return f"Provider request timed out after {read_timeout_s:g}s."diff --git a/providers/base.py b/providers/base.pyindex caacce21..72f1a384 100644--- a/providers/base.py+++ b/providers/base.py@@ -96,11 +96,13 @@ class BaseProvider(ABC):from loguru import loggerfrom core.trace import trace_event+ from providers.error_mapping import exception_cause_typesresponse = getattr(error, "response", None)http_status = (getattr(response, "status_code", None) if response is not None else None)+ cause_types = exception_cause_types(error)trace_event(stage="provider",event="provider.response.transport_error",@@ -109,19 +111,21 @@ class BaseProvider(ABC):request_id=request_id,exc_type=type(error).__name__,http_status=http_status,+ cause_types=cause_types,)if self._config.log_api_error_tracebacks:- logger.error(+ logger.opt(exception=error).error("{}_ERROR:{} {}: {}", tag, req_tag, type(error).__name__, error)returnlogger.error(- "{}_ERROR:{} exc_type={} http_status={}",+ "{}_ERROR:{} exc_type={} http_status={} cause_types={}",tag,req_tag,type(error).__name__,http_status,+ ",".join(cause_types) if cause_types else None,)@abstractmethoddiff --git a/providers/error_mapping.py b/providers/error_mapping.pyindex 35c8d7fc..a5fa6b4a 100644--- a/providers/error_mapping.py+++ b/providers/error_mapping.py@@ -1,6 +1,7 @@"""Provider-specific exception mapping and user-visible diagnostics."""import json+import refrom dataclasses import dataclassfrom typing import Any@@ -24,6 +25,12 @@ from providers.rate_limit import GlobalRateLimiter_BODY_ATTR = "_fcc_provider_error_body"_BODY_TRUNCATED_ATTR = "_fcc_provider_error_body_truncated"+_MAX_CAUSE_CHAIN_DEPTH = 4+_SECRET_TEXT_PATTERNS = (+ re.compile(r"(?i)(authorization\s*[:=]\s*)(bearer\s+)?[^\s,;]+"),+ re.compile(r"(?i)((?:api[_-]?key|token|secret)\s*[:=]\s*)[^\s,;]+"),+ re.compile(r"(?i)(bearer\s+)[^\s,;]+"),+)@dataclass(frozen=True)@@ -33,6 +40,7 @@ class ProviderErrorDetail:status_code: int | None = Nonebody_text: str | None = Noneexception_text: str | None = None+ cause_chain_text: str | None = Noneerror_type_hint: str | None = Nonebody_truncated: bool = False@@ -100,6 +108,49 @@ def _cap_text_bytes(text: str, max_bytes: int) -> tuple[str, bool]:return f"{capped}\n... [truncated after {max_bytes} bytes]", True+def _sanitize_exception_text(text: str) -> str:+ sanitized = text+ for pattern in _SECRET_TEXT_PATTERNS:+ sanitized = pattern.sub(r"\1<redacted>", sanitized)+ return sanitized+++def _exception_causes(exc: BaseException) -> tuple[BaseException, ...]:+ causes: list[BaseException] = []+ seen: set[int] = {id(exc)}+ current: BaseException | None = exc+ while current is not None and len(causes) < _MAX_CAUSE_CHAIN_DEPTH:+ next_exc = current.__cause__ or current.__context__+ if next_exc is None or id(next_exc) in seen:+ break+ seen.add(id(next_exc))+ causes.append(next_exc)+ current = next_exc+ return tuple(causes)+++def exception_cause_types(exc: BaseException) -> tuple[str, ...]:+ """Return safe exception cause type names for default metadata logging."""+ return tuple(type(cause).__name__ for cause in _exception_causes(exc))+++def _exception_cause_chain_text(exc: BaseException) -> str | None:+ lines: list[str] = []+ for cause in _exception_causes(exc):+ raw_text = str(cause).strip()+ if raw_text:+ lines.append(+ f"{type(cause).__name__}: {_sanitize_exception_text(raw_text)}"+ )+ else:+ lines.append(type(cause).__name__)+ if not lines:+ return None+ text = "\n".join(lines)+ capped, _ = _cap_text_bytes(text, PROVIDER_ERROR_BODY_DISPLAY_CAP_BYTES)+ return capped++def _error_type_hint_from_body(body: Any, body_text: str | None) -> str | None:parsed = bodyif isinstance(parsed, bytes):@@ -152,6 +203,7 @@ def extract_provider_error_detail(exc: Exception) -> ProviderErrorDetail:exception_text = str(exc).strip() or Noneif exception_text is not None:+ exception_text = _sanitize_exception_text(exception_text)exception_text, _ = _cap_text_bytes(exception_text, PROVIDER_ERROR_BODY_DISPLAY_CAP_BYTES)@@ -160,6 +212,7 @@ def extract_provider_error_detail(exc: Exception) -> ProviderErrorDetail:status_code=_status_code_from_exception(exc),body_text=body_text,exception_text=exception_text,+ cause_chain_text=_exception_cause_chain_text(exc),error_type_hint=_error_type_hint_from_body(raw_body, body_text),body_truncated=display_truncated,)@@ -194,6 +247,8 @@ def format_provider_error_message(lines = [stable_message]if detail.exception_text and detail.exception_text != stable_message:lines.extend(("", "Provider exception:", detail.exception_text))+ if detail.cause_chain_text:+ lines.extend(("", "Caused by:", detail.cause_chain_text))_append_request_id_lines(lines, request_id)return "\n".join(lines)diff --git a/providers/rate_limit.py b/providers/rate_limit.pyindex 034ffd36..3c2a0504 100644--- a/providers/rate_limit.py+++ b/providers/rate_limit.py@@ -7,6 +7,8 @@ from collections.abc import AsyncIterator, Callablefrom contextlib import asynccontextmanagerfrom typing import Any, ClassVar, TypeVar+import httpx+import openaifrom loguru import loggerfrom core.anthropic.streaming import retryable_transient_status@@ -36,6 +38,26 @@ def retryable_upstream_status(exc: BaseException) -> int | None:return None+def retryable_upstream_transport_error(exc: BaseException) -> bool:+ """Return whether a pre-response transport failure can be retried."""+ if isinstance(exc, openai.AuthenticationError | openai.BadRequestError):+ return False+ return isinstance(+ exc,+ (+ TimeoutError,+ httpx.TimeoutException,+ httpx.ConnectError,+ httpx.ReadError,+ httpx.WriteError,+ httpx.RemoteProtocolError,+ httpx.NetworkError,+ openai.APITimeoutError,+ openai.APIConnectionError,+ ),+ )++class GlobalRateLimiter:"""Global singleton rate limiter that blocks all requests@@ -227,7 +249,9 @@ class GlobalRateLimiter:Waits for the proactive limiter before each attempt. On ``429`` (rate limit)or upstream ``5xx`` server errors, applies exponential backoff with jitter- and sets the reactive block before retrying.+ and sets the reactive block before retrying. Pre-response transport errors+ use the same attempt budget and backoff schedule without setting the+ reactive provider block.Args:fn: Async callable to execute.@@ -252,14 +276,20 @@ class GlobalRateLimiter:return await fn(*args, **kwargs)except Exception as e:status = retryable_upstream_status(e)- if status is None:+ transport_error = status is None and retryable_upstream_transport_error(+ e+ )+ if status is None and not transport_error:raise- label = (- "Rate limited (429)"- if status == 429- else f"Upstream server error ({status})"- )+ if status is None:+ label = f"Provider transport error ({type(e).__name__})"+ else:+ label = (+ "Rate limited (429)"+ if status == 429+ else f"Upstream server error ({status})"+ )last_exc = eif attempt >= max_retries:logger.warning(@@ -285,11 +315,13 @@ class GlobalRateLimiter:event="provider.retry.scheduled",source="provider",status_code=status,+ exc_type=type(e).__name__,attempt=attempt_no,max_attempts=total_attempts,delay_s=round(delay, 3),)- self.set_blocked(delay)+ if status is not None:+ self.set_blocked(delay)await asyncio.sleep(delay)assert last_exc is not Nonediff --git a/providers/transports/anthropic_messages/stream.py b/providers/transports/anthropic_messages/stream.pyindex 9f460870..2a3f9cd1 100644--- a/providers/transports/anthropic_messages/stream.py+++ b/providers/transports/anthropic_messages/stream.py@@ -202,6 +202,7 @@ class AnthropicMessagesStreamAdapter:event="provider.response.error",source="provider",provider=tag,+ request_id=self._request_id,error_message=error_message,exc_type=type(error).__name__,mid_stream=(diff --git a/providers/transports/openai_chat/stream.py b/providers/transports/openai_chat/stream.pyindex e3c4d5c8..be719e63 100644--- a/providers/transports/openai_chat/stream.py+++ b/providers/transports/openai_chat/stream.py@@ -275,6 +275,8 @@ class OpenAIChatStreamAdapter:event="provider.response.error",source="provider",provider=tag,+ request_id=self._request_id,+ exc_type=type(error).__name__,error_message=error_message,mapped_error_type=type(map_error(diff --git a/pyproject.toml b/pyproject.tomlindex a27fc8ff..c36c375b 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "hatchling.build"[project]name = "free-claude-code"-version = "3.4.7"+version = "3.4.8"description = "Middleware between Claude Code CLI (Anthropic API) and NVIDIA NIM"readme = "README.md"requires-python = ">=3.14.0"diff --git a/tests/providers/test_anthropic_messages_429_retry.py b/tests/providers/test_anthropic_messages_429_retry.pyindex d1753dd9..0b68ec9b 100644--- a/tests/providers/test_anthropic_messages_429_retry.py+++ b/tests/providers/test_anthropic_messages_429_retry.py@@ -140,6 +140,53 @@ async def test_native_stream_retries_on_http_5xx_then_streams(GlobalRateLimiter.reset_instance()+async def test_native_stream_retries_on_pre_send_connection_error_then_streams(+ provider_config,+):+ """Pre-response HTTPX transport errors retry through execute_with_retry."""+ GlobalRateLimiter.reset_instance()+ try:+ provider = NativeProvider(provider_config)+ req = MockRequest()+ request_obj = httpx.Request("POST", "https://custom.test/v1/messages")+ ok_lines = [+ "event: message_start",+ 'data: {"type":"message_start"}',+ "",+ "event: message_stop",+ 'data: {"type":"message_stop"}',+ "",+ ]+ ok_response = FakeResponse(lines=ok_lines)++ send_calls = {"n": 0}++ async def send_side_effect(*_a, **_kw):+ send_calls["n"] += 1+ if send_calls["n"] == 1:+ raise httpx.ConnectError("connect failed", request=request_obj)+ return ok_response++ with (+ patch.object(provider._client, "build_request", return_value=request_obj),+ patch.object(+ provider._client,+ "send",+ new_callable=AsyncMock,+ side_effect=send_side_effect,+ ),+ patch("asyncio.sleep", new_callable=AsyncMock),+ ):+ events = [e async for e in provider.stream_response(req)]++ assert send_calls["n"] == 2+ assert ok_response.is_closed+ _assert_minimal_success_stream(events)+ finally:+ GlobalRateLimiter.reset_instance()++@pytest.mark.parametrize(("status_code", "substr"),[@@ -202,6 +249,74 @@ async def test_native_stream_5xx_retry_exhausted(provider_config, status_code, sGlobalRateLimiter.reset_instance()+async def test_native_stream_connection_error_retry_exhausted(provider_config):+ """Repeated pre-response connection failures exhaust at 5 attempts."""+ GlobalRateLimiter.reset_instance()+ try:++ @asynccontextmanager+ async def _slot():+ yield++ with patch(+ "providers.transports.anthropic_messages.transport.GlobalRateLimiter"+ ) as mock_gl:+ instance = mock_gl.get_scoped_instance.return_value+ real = GlobalRateLimiter(+ rate_limit=100,+ rate_window=60,+ max_concurrency=5,+ )+ instance.wait_if_blocked = real.wait_if_blocked+ instance.execute_with_retry = real.execute_with_retry+ instance.set_blocked = real.set_blocked+ instance.concurrency_slot.side_effect = _slot++ provider = NativeProvider(provider_config)+ req = MockRequest()+ request_obj = httpx.Request("POST", "https://custom.test/v1/messages")++ with (+ patch.object(+ provider._client, "build_request", return_value=request_obj+ ),+ patch.object(+ provider._client,+ "send",+ new_callable=AsyncMock,+ side_effect=httpx.ConnectError(+ "connect failed", request=request_obj+ ),+ ) as mock_send,+ patch("asyncio.sleep", new_callable=AsyncMock),+ patch(+ "providers.transports.anthropic_messages.stream.trace_event"+ ) as trace,+ ):+ events = [+ e+ async for e in provider.stream_response(+ req, request_id="req_native_conn"+ )+ ]++ assert mock_send.await_count == 5+ error_traces = [+ call.kwargs+ for call in trace.call_args_list+ if call.kwargs.get("event") == "provider.response.error"+ ]+ assert error_traces[-1]["request_id"] == "req_native_conn"+ assert error_traces[-1]["exc_type"] == "ConnectError"+ assert_canonical_stream_error_envelope(+ events,+ user_message_substr="Provider exception:\nconnect failed",+ )+ finally:+ GlobalRateLimiter.reset_instance()++@pytest.mark.asyncioasync def test_non_retryable_4xx_http_error_not_retried(provider_config):"""HTTP 400 from upstream is not retried; single send (passthrough limiter)."""diff --git a/tests/providers/test_error_mapping.py b/tests/providers/test_error_mapping.pyindex e740225a..a953b16a 100644--- a/tests/providers/test_error_mapping.py+++ b/tests/providers/test_error_mapping.py@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patchimport openaiimport pytest-from httpx import HTTPStatusError, ReadTimeout, Request, Response+from httpx import ConnectError, HTTPStatusError, ReadTimeout, Request, Responsefrom config.constants import PROVIDER_ERROR_BODY_DISPLAY_CAP_BYTESfrom core.anthropic import (@@ -15,6 +15,7 @@ from core.anthropic import ()from providers.error_mapping import (attach_provider_error_body,+ exception_cause_types,extract_provider_error_detail,format_provider_error_message,map_error,@@ -313,6 +314,48 @@ def test_empty_http_error_body_is_explicitly_reported():assert "(empty upstream error body)" in msg+def test_connection_error_without_response_includes_sanitized_cause_chain():+ request = Request("POST", "http://test")+ exc = openai.APIConnectionError(request=request)+ exc.__cause__ = ConnectError(+ "connect failed authorization: Bearer SECRET token=ALSO_SECRET",+ request=request,+ )+ mapped = map_error(exc)+ detail = extract_provider_error_detail(exc)+ msg = format_provider_error_message(+ mapped,+ detail,+ provider_name="NIM",+ read_timeout_s=30.0,+ request_id="req_conn",+ )++ assert "Provider API request failed." in msg+ assert "Provider exception:" in msg+ assert "Connection error." in msg+ assert "Caused by:" in msg+ assert "ConnectError: connect failed authorization: <redacted>" in msg+ assert "SECRET" not in msg+ assert "Request ID: req_conn" in msg+ assert exception_cause_types(exc) == ("ConnectError",)+++def test_connection_error_cause_chain_is_capped_for_display():+ request = Request("POST", "http://test")+ exc = openai.APIConnectionError(request=request)+ exc.__cause__ = ConnectError(+ "x" * (PROVIDER_ERROR_BODY_DISPLAY_CAP_BYTES + 10),+ request=request,+ )+ detail = extract_provider_error_detail(exc)++ assert detail.cause_chain_text is not None+ assert f"truncated after {PROVIDER_ERROR_BODY_DISPLAY_CAP_BYTES} bytes" in (+ detail.cause_chain_text+ )++def test_attached_provider_error_body_is_capped_for_display():response = Response(status_code=500,diff --git a/tests/providers/test_llamacpp.py b/tests/providers/test_llamacpp.pyindex a2d8e7b2..bfc4333b 100644--- a/tests/providers/test_llamacpp.py+++ b/tests/providers/test_llamacpp.py@@ -290,8 +290,10 @@ async def test_stream_network_error(llamacpp_provider):blob = "".join(events)assert_canonical_stream_error_envelope(- events, user_message_substr="Connection refused"+ events, user_message_substr="Could not connect to provider.")+ assert "Provider exception" in blob+ assert "Connection refused" in blobassert "TEST_ID2" in blobdiff --git a/tests/providers/test_openai_compat_5xx_retry.py b/tests/providers/test_openai_compat_5xx_retry.pyindex a17c2a3f..76fbb749 100644--- a/tests/providers/test_openai_compat_5xx_retry.py+++ b/tests/providers/test_openai_compat_5xx_retry.py@@ -2,6 +2,7 @@from unittest.mock import AsyncMock, MagicMock, patch+import httpximport openaiimport pytestfrom httpx import Request, Response@@ -11,6 +12,7 @@ from providers.base import ProviderConfigfrom providers.nvidia_nim import NvidiaNimProviderfrom providers.rate_limit import GlobalRateLimiterfrom tests.providers.test_nvidia_nim import MockRequest+from tests.stream_contract import assert_canonical_stream_error_envelopedef _internal_5xx(code: int) -> openai.InternalServerError:@@ -21,6 +23,14 @@ def _internal_5xx(code: int) -> openai.InternalServerError:)+def _connection_error(message: str = "connect failed") -> openai.APIConnectionError:+ error = openai.APIConnectionError(+ request=Request("POST", "https://test.api.nvidia.com/v1/chat/completions")+ )+ error.__cause__ = httpx.ConnectError(message)+ return error++@pytest.mark.parametrize("status_code", [500, 502, 503, 504])@pytest.mark.asyncioasync def test_nim_stream_retries_on_openai_5xx_then_streams(status_code):@@ -67,6 +77,98 @@ async def test_nim_stream_retries_on_openai_5xx_then_streams(status_code):GlobalRateLimiter.reset_instance()+async def test_nim_stream_retries_on_pre_stream_connection_error_then_streams():+ GlobalRateLimiter.reset_instance()+ try:+ config = ProviderConfig(+ api_key="test_key",+ base_url="https://test.api.nvidia.com/v1",+ rate_limit=100,+ rate_window=60,+ http_read_timeout=600.0,+ http_write_timeout=15.0,+ http_connect_timeout=5.0,+ )+ provider = NvidiaNimProvider(config, nim_settings=NimSettings())+ req = MockRequest()++ mock_chunk = MagicMock()+ mock_chunk.choices = [+ MagicMock(+ delta=MagicMock(content="Recovered", reasoning_content=""),+ finish_reason="stop",+ )+ ]+ mock_chunk.usage = None++ async def mock_stream():+ yield mock_chunk++ with (+ patch.object(+ provider._client.chat.completions,+ "create",+ new_callable=AsyncMock,+ ) as mock_create,+ patch("asyncio.sleep", new_callable=AsyncMock),+ ):+ mock_create.side_effect = [_connection_error(), mock_stream()]+ events = [e async for e in provider.stream_response(req)]++ assert mock_create.await_count == 2+ assert any("Recovered" in e for e in events)+ finally:+ GlobalRateLimiter.reset_instance()+++async def test_nim_stream_connection_error_exhausted_emits_cause_chain():+ GlobalRateLimiter.reset_instance()+ try:+ config = ProviderConfig(+ api_key="test_key",+ base_url="https://test.api.nvidia.com/v1",+ rate_limit=100,+ rate_window=60,+ http_read_timeout=600.0,+ http_write_timeout=15.0,+ http_connect_timeout=5.0,+ )+ provider = NvidiaNimProvider(config, nim_settings=NimSettings())+ req = MockRequest()+ error = _connection_error("upstream disconnected")++ with (+ patch.object(+ provider._client.chat.completions,+ "create",+ new_callable=AsyncMock,+ side_effect=error,+ ) as mock_create,+ patch("asyncio.sleep", new_callable=AsyncMock),+ patch("providers.transports.openai_chat.stream.trace_event") as trace,+ ):+ events = [+ e async for e in provider.stream_response(req, request_id="req_conn")+ ]++ assert mock_create.await_count == 5+ error_traces = [+ call.kwargs+ for call in trace.call_args_list+ if call.kwargs.get("event") == "provider.response.error"+ ]+ assert error_traces[-1]["request_id"] == "req_conn"+ assert error_traces[-1]["exc_type"] == "APIConnectionError"+ assert_canonical_stream_error_envelope(+ events,+ user_message_substr="Caused by:\nConnectError: upstream disconnected",+ )+ finally:+ GlobalRateLimiter.reset_instance()++@pytest.mark.parametrize(("status_code", "expect_substr"),[diff --git a/tests/providers/test_provider_rate_limit.py b/tests/providers/test_provider_rate_limit.pyindex f46b8bf1..e8658058 100644--- a/tests/providers/test_provider_rate_limit.py+++ b/tests/providers/test_provider_rate_limit.py@@ -1,6 +1,8 @@import asyncioimport time+from unittest.mock import AsyncMock, patch+import httpximport openaiimport pytestimport pytest_asyncio@@ -11,6 +13,7 @@ from providers.rate_limit import (UPSTREAM_TRANSIENT_TOTAL_ATTEMPTS,GlobalRateLimiter,retryable_upstream_status,+ retryable_upstream_transport_error,)@@ -41,6 +44,40 @@ def test_retryable_upstream_status_reads_statusless_resource_exhausted_text() ->assert retryable_upstream_status(exc) == 503+def test_retryable_upstream_transport_error_classifies_connection_failures() -> None:+ request = Request("POST", "http://x")+ assert retryable_upstream_transport_error(+ openai.APIConnectionError(request=request)+ )+ assert retryable_upstream_transport_error(httpx.ConnectError("connect failed"))+ assert retryable_upstream_transport_error(httpx.ReadError("read failed"))+ assert retryable_upstream_transport_error(httpx.WriteError("write failed"))+ assert retryable_upstream_transport_error(+ httpx.RemoteProtocolError("server disconnected")+ )+ assert retryable_upstream_transport_error(TimeoutError("timed out"))+++def test_retryable_upstream_transport_error_rejects_request_errors() -> None:+ from httpx import Response++ request = Request("POST", "http://x")+ assert not retryable_upstream_transport_error(+ openai.BadRequestError(+ "bad request",+ response=Response(400, request=request),+ body={},+ )+ )+ assert not retryable_upstream_transport_error(+ httpx.HTTPStatusError(+ "bad request",+ request=request,+ response=Response(400, request=request),+ )+ )++class TestProviderRateLimiter:"""Tests for providers.rate_limit.GlobalRateLimiter."""@@ -399,6 +436,97 @@ class TestProviderRateLimiter:assert result == "ok"assert call_count == 2+ @pytest.mark.asyncio+ async def test_execute_with_retry_succeeds_on_openai_connection_retry(self):+ """Pre-response OpenAI SDK connection errors retry then succeed."""+ limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)++ call_count = 0+ request = Request("POST", "http://x")++ async def fail_then_ok():+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ raise openai.APIConnectionError(request=request)+ return "ok"++ with (+ patch.object(limiter, "set_blocked") as set_blocked,+ patch("asyncio.sleep", new_callable=AsyncMock) as sleep,+ ):+ result = await limiter.execute_with_retry(+ fail_then_ok,+ max_retries=2,+ base_delay=0.01,+ max_delay=0.1,+ jitter=0,+ )++ assert result == "ok"+ assert call_count == 2+ set_blocked.assert_not_called()+ sleep.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_execute_with_retry_succeeds_on_httpx_transport_retry(self):+ """Pre-response HTTPX transport errors retry then succeed."""+ limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)++ call_count = 0++ async def fail_then_ok():+ nonlocal call_count+ call_count += 1+ if call_count == 1:+ raise httpx.RemoteProtocolError("server disconnected")+ return "ok"++ with (+ patch.object(limiter, "set_blocked") as set_blocked,+ patch("asyncio.sleep", new_callable=AsyncMock) as sleep,+ ):+ result = await limiter.execute_with_retry(+ fail_then_ok,+ max_retries=2,+ base_delay=0.01,+ max_delay=0.1,+ jitter=0,+ )++ assert result == "ok"+ assert call_count == 2+ set_blocked.assert_not_called()+ sleep.assert_awaited_once()++ @pytest.mark.asyncio+ async def test_execute_with_retry_exhaust_transport_error_attempts(self):+ """Transport retries exhaust after the shared 5 total attempts."""+ limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)++ call_count = 0++ async def always_fail():+ nonlocal call_count+ call_count += 1+ raise httpx.ConnectError("connect failed")++ with (+ patch.object(limiter, "set_blocked") as set_blocked,+ patch("asyncio.sleep", new_callable=AsyncMock) as sleep,+ pytest.raises(httpx.ConnectError),+ ):+ await limiter.execute_with_retry(+ always_fail,+ base_delay=0.01,+ max_delay=0.1,+ jitter=0,+ )++ assert call_count == UPSTREAM_TRANSIENT_TOTAL_ATTEMPTS+ set_blocked.assert_not_called()+ assert sleep.await_count == DEFAULT_UPSTREAM_MAX_RETRIES+@pytest.mark.parametrize("status_code", [500, 502, 503, 504])@pytest.mark.asyncioasync def test_execute_with_retry_exhaust_openai_5xx_raises(self, status_code):diff --git a/tests/providers/test_provider_transport_logging.py b/tests/providers/test_provider_transport_logging.pyindex ecc526bf..6141dd41 100644--- a/tests/providers/test_provider_transport_logging.py+++ b/tests/providers/test_provider_transport_logging.py@@ -4,6 +4,8 @@ import loggingfrom contextlib import asynccontextmanagerfrom unittest.mock import AsyncMock, MagicMock, patch+import httpx+import openaiimport pytestfrom config.constants import NATIVE_MESSAGES_ERROR_BODY_LOG_CAP_BYTES@@ -231,6 +233,46 @@ async def test_openai_compat_stream_failure_default_logs_exclude_exception_str(cassert "exc_type=RuntimeError" in messages+async def test_openai_compat_stream_failure_default_logs_cause_types_only(caplog):+ config = ProviderConfig(+ api_key="k",+ base_url="http://localhost:1/v1",+ log_api_error_tracebacks=False,+ )+ provider = NvidiaNimProvider(config, nim_settings=NimSettings())+ req = make_openai_compat_stream_request()+ error = openai.APIConnectionError(+ request=httpx.Request("POST", "http://localhost:1/v1/chat/completions")+ )+ error.__cause__ = httpx.ConnectError("SECRET_CAUSE_DETAIL")++ @asynccontextmanager+ async def _noop_slot():+ yield++ with (+ patch.object(+ provider,+ "_create_stream",+ new_callable=AsyncMock,+ side_effect=error,+ ),+ patch.object(+ provider._global_rate_limiter,+ "concurrency_slot",+ _noop_slot,+ ),+ caplog.at_level(logging.ERROR),+ ):+ _ = [e async for e in provider.stream_response(req)]++ messages = " | ".join(r.getMessage() for r in caplog.records)+ assert "SECRET_CAUSE_DETAIL" not in messages+ assert "exc_type=APIConnectionError" in messages+ assert "cause_types=ConnectError" in messages++@pytest.mark.asyncioasync def test_openai_compat_stream_failure_respects_verbose_flag(caplog):config = ProviderConfig(diff --git a/uv.lock b/uv.lockindex 792e7cf8..192d4b36 100644--- a/uv.lock+++ b/uv.lock@@ -561,7 +561,7 @@ wheels = [[[package]]name = "free-claude-code"-version = "3.4.7"+version = "3.4.8"source = { editable = "." }dependencies = [{ name = "aiohttp" },