Your lightweight open-source personal agent.
Python agent-frameworkai-agentai-agentschatbotchatopsdiscord-botllm-agentsllmslocal-llmmcpmodel-context-protocolmulti-agentopenai-compatibleopenclawpersonal-ai-assistantpythonself-hostedtelegram-bot-ai-assistantwebuiworkflow-automation
fix(retry): prevent replay amplification and isolate backoff (NAN-313)
chengyongru4e77d9c3f2bc
diff --git a/nanobot/agent/tools/mcp.py b/nanobot/agent/tools/mcp.pyindex 87d92259e..197878b80 100644--- a/nanobot/agent/tools/mcp.py+++ b/nanobot/agent/tools/mcp.py@@ -12,7 +12,7 @@ import tracebackimport urllib.parsefrom collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mappingfrom contextlib import AsyncExitStack, suppress-from typing import TYPE_CHECKING, Any, Literal, Protocol, cast+from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar, castimport httpxfrom loguru import logger@@ -62,6 +62,7 @@ _HTTP_READ_TIMEOUT = 300.0# Replace anything outside [a-zA-Z0-9_-] with underscore and collapse runs._SANITIZE_RE = re.compile(r"_+")_ReconnectCallback = Callable[[str, str, Tool], Awaitable[Tool | None]]+_MCPResultT = TypeVar("_MCPResultT")MCPServerLoader = Callable[[], tuple[Mapping[str, "MCPServerConfig"], NetworkConfig]]MCPRuntimeStatus = Literal["connecting", "connected", "failed"]@@ -572,6 +573,75 @@ class _MCPWrapperBase(Tool):self._session = refreshed_sessionreturn True+ async def _request_with_recovery(+ self,+ request: Callable[[], Awaitable[_MCPResultT]],+ timeout: float,+ capability_kind: Literal["tool", "resource", "prompt"],+ ) -> _MCPResultT | ToolResult:+ """Recover requests only; rendering and reconnect/backoff failures stay outside replay."""+ from mcp.shared.exceptions import McpError++ operation = f"{capability_kind} {'read' if capability_kind == 'resource' else 'call'}"+ refreshed_session = False+ retried = False+ while True:+ try:+ return await asyncio.wait_for(request(), timeout=timeout)+ except asyncio.TimeoutError:+ logger.warning(+ "MCP {} '{}' timed out after {}s", capability_kind, self._name, timeout+ )+ return ToolResult.error(f"(MCP {operation} timed out after {timeout}s)")+ except asyncio.CancelledError:+ # MCP SDK cancel scopes can leak cancellation without cancelling our task.+ if task_is_cancelling():+ raise+ logger.warning("MCP {} '{}' was cancelled by server/SDK", capability_kind, self._name)+ return ToolResult.error(f"(MCP {operation} was cancelled)")+ except Exception as exc:+ if await self._refresh_session_after_termination(+ exc, refreshed_session, capability_kind+ ):+ refreshed_session = True+ continue+ if capability_kind == "prompt" and isinstance(exc, McpError):+ logger.opt(exception=exc if tool_log_content_allowed() else False).error(+ "MCP prompt '{}' failed: code={} message={}",+ self._name,+ exc.error.code,+ exc.error.message if tool_log_content_allowed() else "[content hidden]",+ )+ return ToolResult.error(+ f"(MCP prompt call failed: {exc.error.message} [code {exc.error.code}])"+ )+ if _is_transient(exc):+ if not retried:+ logger.warning(+ "MCP {} '{}' hit transient error ({}), retrying once...",+ capability_kind,+ self._name,+ type(exc).__name__,+ )+ retried = True+ await asyncio.sleep(1)+ continue+ logger.opt(exception=exc if tool_log_content_allowed() else False).error(+ "MCP {} '{}' failed after retry: {}",+ capability_kind,+ self._name,+ type(exc).__name__,+ )+ return ToolResult.error(f"(MCP {operation} failed after retry: {type(exc).__name__})")+ logger.opt(exception=exc if tool_log_content_allowed() else False).error(+ "MCP {} '{}' failed: {}: {}",+ capability_kind,+ self._name,+ type(exc).__name__,+ exc if tool_log_content_allowed() else "[content hidden]",+ )+ return ToolResult.error(f"(MCP {operation} failed: {type(exc).__name__})")+def _image_block_data_url(block: Any, types: Any) -> str | None:"""Return a base64 ``data:`` URL for an MCP image-bearing content block.@@ -653,81 +723,29 @@ class MCPToolWrapper(_MCPWrapperBase):return self._parametersasync def execute(self, **kwargs: Any) -> str:- retried_transient = False- refreshed_session = False- while True:- try:- result = await asyncio.wait_for(- self._session.call_tool(self._original_name, arguments=kwargs),- timeout=self._tool_timeout,- )- except asyncio.TimeoutError:- logger.warning(- "MCP tool '{}' timed out after {}s", self._name, self._tool_timeout- )- return ToolResult.error(- f"(MCP tool call timed out after {self._tool_timeout}s)"- )- except asyncio.CancelledError:- # MCP SDK's anyio cancel scopes can leak CancelledError on timeout/failure.- # Re-raise only if our task was externally cancelled (e.g. /stop).- if task_is_cancelling():- raise- logger.warning("MCP tool '{}' was cancelled by server/SDK", self._name)- return ToolResult.error("(MCP tool call was cancelled)")- except Exception as exc:- if await self._refresh_session_after_termination(- exc,- refreshed_session,- "tool",- ):- refreshed_session = True- continue- if _is_transient(exc):- if not retried_transient:- retried_transient = True- logger.warning(- "MCP tool '{}' hit transient error ({}), retrying once...",- self._name,- type(exc).__name__,- )- await asyncio.sleep(1) # Brief backoff before retry- continue- # Second transient failure — give up with retry-specific message- logger.opt(exception=tool_log_content_allowed()).error(- "MCP tool '{}' failed after retry: {}",- self._name,- type(exc).__name__,- )- return ToolResult.error(- f"(MCP tool call failed after retry: {type(exc).__name__})"- )- logger.opt(exception=tool_log_content_allowed()).error(- "MCP tool '{}' failed: {}: {}",- self._name,- type(exc).__name__,- exc if tool_log_content_allowed() else "[content hidden]",- )- return ToolResult.error(- f"(MCP tool call failed: {type(exc).__name__})"- )- else:- # Success — extract text and persist any image content as artifacts.- try:- rendered = self._render_call_result(result.content, kwargs)- if getattr(result, "isError", False):- return ToolResult.error(rendered)- return rendered- except Exception as exc:- logger.opt(exception=tool_log_content_allowed()).error(- "MCP tool '{}' failed while rendering result: {}: {}",- self._name,- type(exc).__name__,- exc if tool_log_content_allowed() else "[content hidden]",- )- return ToolResult.error(- f"(MCP tool returned malformed content: {type(exc).__name__})"- )+ result = await self._request_with_recovery(+ lambda: self._session.call_tool(self._original_name, arguments=kwargs),+ self._tool_timeout,+ "tool",+ )+ if isinstance(result, ToolResult):+ return result+ # Success — extract text and persist any image content as artifacts.+ try:+ rendered = self._render_call_result(result.content, kwargs)+ if getattr(result, "isError", False):+ return ToolResult.error(rendered)+ return rendered+ except Exception as exc:+ logger.opt(exception=tool_log_content_allowed()).error(+ "MCP tool '{}' failed while rendering result: {}: {}",+ self._name,+ type(exc).__name__,+ exc if tool_log_content_allowed() else "[content hidden]",+ )+ return ToolResult.error(+ f"(MCP tool returned malformed content: {type(exc).__name__})"+ )def _render_call_result(self, content: Any, arguments: Mapping[str, Any]) -> str:"""Turn MCP content blocks into a tool result string.@@ -825,65 +843,22 @@ class MCPResourceWrapper(_MCPWrapperBase):async def execute(self, **kwargs: Any) -> str:from mcp import types- retried_transient = False- refreshed_session = False- while True:- try:- result = await asyncio.wait_for(- self._session.read_resource(self._uri),- timeout=self._resource_timeout,- )- except asyncio.TimeoutError:- logger.warning(- "MCP resource '{}' timed out after {}s", self._name, self._resource_timeout- )- return f"(MCP resource read timed out after {self._resource_timeout}s)"- except asyncio.CancelledError:- if task_is_cancelling():- raise- logger.warning("MCP resource '{}' was cancelled by server/SDK", self._name)- return "(MCP resource read was cancelled)"- except Exception as exc:- if await self._refresh_session_after_termination(- exc,- refreshed_session,- "resource",- ):- refreshed_session = True- continue- if _is_transient(exc):- if not retried_transient:- retried_transient = True- logger.warning(- "MCP resource '{}' hit transient error ({}), retrying once...",- self._name,- type(exc).__name__,- )- await asyncio.sleep(1)- continue- logger.opt(exception=tool_log_content_allowed()).error(- "MCP resource '{}' failed after retry: {}",- self._name,- type(exc).__name__,- )- return f"(MCP resource read failed after retry: {type(exc).__name__})"- logger.opt(exception=tool_log_content_allowed()).error(- "MCP resource '{}' failed: {}: {}",- self._name,- type(exc).__name__,- exc if tool_log_content_allowed() else "[content hidden]",- )- return f"(MCP resource read failed: {type(exc).__name__})"+ result = await self._request_with_recovery(+ lambda: self._session.read_resource(self._uri),+ self._resource_timeout,+ "resource",+ )+ if isinstance(result, ToolResult):+ return str(result)+ parts: list[str] = []+ for block in result.contents:+ if isinstance(block, types.TextResourceContents):+ parts.append(block.text)+ elif isinstance(cast(object, block), types.BlobResourceContents):+ parts.append(f"[Binary resource: {len(block.blob)} bytes]")else:- parts: list[str] = []- for block in result.contents:- if isinstance(block, types.TextResourceContents):- parts.append(block.text)- elif isinstance(cast(object, block), types.BlobResourceContents):- parts.append(f"[Binary resource: {len(block.blob)} bytes]")- else:- parts.append(str(block))- return "\n".join(parts) or "(no output)"+ parts.append(str(block))+ return "\n".join(parts) or "(no output)"class MCPPromptWrapper(_MCPWrapperBase):@@ -942,87 +917,28 @@ class MCPPromptWrapper(_MCPWrapperBase):async def execute(self, **kwargs: Any) -> str:from mcp import types- from mcp.shared.exceptions import McpError- retried_transient = False- refreshed_session = False- while True:- try:- result = await asyncio.wait_for(- self._session.get_prompt(self._prompt_name, arguments=kwargs),- timeout=self._prompt_timeout,- )- except asyncio.TimeoutError:- logger.warning(- "MCP prompt '{}' timed out after {}s", self._name, self._prompt_timeout- )- return f"(MCP prompt call timed out after {self._prompt_timeout}s)"- except asyncio.CancelledError:- if task_is_cancelling():- raise- logger.warning("MCP prompt '{}' was cancelled by server/SDK", self._name)- return "(MCP prompt call was cancelled)"- except McpError as exc:- if await self._refresh_session_after_termination(- exc,- refreshed_session,- "prompt",- ):- refreshed_session = True- continue- logger.opt(exception=tool_log_content_allowed()).error(- "MCP prompt '{}' failed: code={} message={}",- self._name,- exc.error.code,- exc.error.message if tool_log_content_allowed() else "[content hidden]",- )- return f"(MCP prompt call failed: {exc.error.message} [code {exc.error.code}])"- except Exception as exc:- if await self._refresh_session_after_termination(- exc,- refreshed_session,- "prompt",- ):- refreshed_session = True- continue- if _is_transient(exc):- if not retried_transient:- retried_transient = True- logger.warning(- "MCP prompt '{}' hit transient error ({}), retrying once...",- self._name,- type(exc).__name__,- )- await asyncio.sleep(1)- continue- logger.opt(exception=tool_log_content_allowed()).error(- "MCP prompt '{}' failed after retry: {}",- self._name,- type(exc).__name__,- )- return f"(MCP prompt call failed after retry: {type(exc).__name__})"- logger.opt(exception=tool_log_content_allowed()).error(- "MCP prompt '{}' failed: {}: {}",- self._name,- type(exc).__name__,- exc if tool_log_content_allowed() else "[content hidden]",- )- return f"(MCP prompt call failed: {type(exc).__name__})"- else:- parts: list[str] = []- for message in result.messages:- content = message.content- if isinstance(content, types.TextContent):- parts.append(content.text)- elif isinstance(content, list):- for block in content:- if isinstance(block, types.TextContent):- parts.append(block.text)- else:- parts.append(str(block))+ result = await self._request_with_recovery(+ lambda: self._session.get_prompt(self._prompt_name, arguments=kwargs),+ self._prompt_timeout,+ "prompt",+ )+ if isinstance(result, ToolResult):+ return str(result)+ parts: list[str] = []+ for message in result.messages:+ content = message.content+ if isinstance(content, types.TextContent):+ parts.append(content.text)+ elif isinstance(content, list):+ for block in content:+ if isinstance(block, types.TextContent):+ parts.append(block.text)else:- parts.append(str(content))- return "\n".join(parts) or "(no output)"+ parts.append(str(block))+ else:+ parts.append(str(content))+ return "\n".join(parts) or "(no output)"async def connect_mcp_servers(diff --git a/nanobot/channels/manager.py b/nanobot/channels/manager.pyindex d6fe97d97..c7fc12476 100644--- a/nanobot/channels/manager.py+++ b/nanobot/channels/manager.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport asyncioimport hashlibimport inspect+import randomimport timefrom collections import OrderedDictfrom collections.abc import Awaitable, Callable, Iterable, Mapping@@ -62,7 +63,7 @@ def _default_webui_dist() -> Path | None:return candidate if candidate.is_dir() else None-# Retry delays for message sending (exponential backoff: 1s, 2s, 4s)+# Upper bounds for local equal-jitter send backoff (1s, 2s, then 4s)._SEND_RETRY_DELAYS = (1, 2, 4)_OUTBOUND_CONCURRENCY = 32_OUTBOUND_PENDING_LIMIT = 256@@ -797,8 +798,8 @@ class ChannelManager:async def _queue_outbound(self, channel: BaseChannel, msg: OutboundMessage) -> None:"""Bound scheduled sends; preserve FIFO per destination, not across chats.- Retry waits occupy only their destination and one send slot. Saturation- pauses dispatch rather than creating unbounded tasks or dropping messages.+ Retry waits retain their destination tail and admission, not a send permit.+ Saturation pauses dispatch rather than creating unbounded tasks or dropping messages."""if msg.delivery_id in self._scheduled_deliveries:return@@ -816,11 +817,12 @@ class ChannelManager:async def send() -> None:if previous is not None:await asyncio.shield(asyncio.gather(previous, return_exceptions=True))- async with self._outbound_sends:- if msg.delivery_id is not None:+ if msg.delivery_id is not None:+ # Acquire before claiming: queued cancellation must remain pending.+ async with self._outbound_sends:await self._send_scheduled(channel, msg)- else:- await self._send_with_retry(channel, msg)+ else:+ await self._send_with_retry(channel, msg)task = asyncio.create_task(send(), name=f"outbound-{msg.channel}-{msg.chat_id}")self._outbound_tasks[task] = key@@ -1168,9 +1170,22 @@ class ChannelManager:attempt = 0while True:- attempt += 1try:- await self._send_once(channel, msg)+ async with asyncio.timeout_at(deadline):+ await self._outbound_sends.acquire()+ except TimeoutError:+ logger.warning("Send to {} deadline expired waiting for a send slot", msg.channel)+ return+ try:+ try:+ if deadline is not None and asyncio.get_running_loop().time() >= deadline:+ logger.warning("Send to {} deadline expired before submission", msg.channel)+ return+ attempt += 1+ # The deadline bounds admission, not an already-submitted send.+ await self._send_once(channel, msg)+ finally:+ self._outbound_sends.release()return # Send succeededexcept asyncio.CancelledError:raise # Propagate cancellation for graceful shutdown@@ -1195,7 +1210,8 @@ class ChannelManager:msg.channel, attempt,)return- delay = _SEND_RETRY_DELAYS[min(attempt - 1, len(_SEND_RETRY_DELAYS) - 1)]+ base = _SEND_RETRY_DELAYS[min(attempt - 1, len(_SEND_RETRY_DELAYS) - 1)]+ delay = random.uniform(base / 2, base)if deadline is not None:delay = min(delay, max(0.0, deadline - loop.time()))attempt_label = str(attempt)diff --git a/nanobot/channels/telegram/runtime.py b/nanobot/channels/telegram/runtime.pyindex fb9fa00ec..2220e54e5 100644--- a/nanobot/channels/telegram/runtime.py+++ b/nanobot/channels/telegram/runtime.py@@ -1136,6 +1136,11 @@ class TelegramChannel(BaseChannel):buf.message_id = sent.message_idbuf.draft_id = None+ def should_retry_send_error(self, error: Exception) -> bool:+ # Per-part API retries belong to this adapter. Whole-message replay is safe+ # only before submission; otherwise it may duplicate a successful prefix.+ return isinstance(error, ChannelNotReadyError)+async def send(self, msg: OutboundMessage) -> None:"""Send a message through Telegram."""if isinstance(msg.event, ContextCompactionEvent) and not (diff --git a/nanobot/channels/weixin/tests/test_weixin_hardening.py b/nanobot/channels/weixin/tests/test_weixin_hardening.pyindex d30e3e80a..917387f83 100644--- a/nanobot/channels/weixin/tests/test_weixin_hardening.py+++ b/nanobot/channels/weixin/tests/test_weixin_hardening.py@@ -118,6 +118,7 @@ async def test_channel_manager_does_not_retry_permanent_weixin_error(monkeypatchmanager = ChannelManager.__new__(ChannelManager)manager.config = Config.model_validate({"channels": {"sendMaxRetries": 3}})manager.bus = MessageBus()+ manager._outbound_sends = asyncio.Semaphore(32)channel = _channel()channel.send = AsyncMock(side_effect=WeixinAPIError(diff --git a/nanobot/channels/whatsapp_agent/tests/test_runtime.py b/nanobot/channels/whatsapp_agent/tests/test_runtime.pyindex 00aece1cb..5b81af546 100644--- a/nanobot/channels/whatsapp_agent/tests/test_runtime.py+++ b/nanobot/channels/whatsapp_agent/tests/test_runtime.py@@ -667,6 +667,7 @@ def test_manager_retry_resumes_after_acknowledged_parts(monkeypatch.setattr("nanobot.channels.manager._SEND_RETRY_DELAYS", (0,))manager = object.__new__(ChannelManager)manager.config = Config()+ manager._outbound_sends = asyncio.Semaphore(32)message = OutboundMessage(channel=channel.name, chat_id="creator", content="A" * 4096 + "B")asyncio.run(manager._send_with_retry(channel, message))assert delivered == ["A" * 4096, "B"]@@ -764,6 +765,7 @@ def test_media_failure_never_replays_accepted_text_or_attachments(monkeypatch.setattr("nanobot.channels.manager._SEND_RETRY_DELAYS", (0,))manager = object.__new__(ChannelManager)manager.config = Config()+ manager._outbound_sends = asyncio.Semaphore(32)message = OutboundMessage(channel=channel.name, chat_id="creator", content="Files",media=[str(attachment), str(attachment)],diff --git a/nanobot/providers/base.py b/nanobot/providers/base.pyindex 429dadbd3..d5a4c5040 100644--- a/nanobot/providers/base.py+++ b/nanobot/providers/base.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport asyncioimport jsonimport os+import randomimport reimport timefrom abc import ABC, abstractmethod@@ -1590,34 +1591,29 @@ class LLMProvider(ABC):should_retry_guard: Callable[[], bool] | None = None,on_stream_recover: Callable[[], Awaitable[None]] | None = None,) -> LLMResponse:- attempt = 0- delays = list(self._CHAT_RETRY_DELAYS)+ response = await call(**kw)+ if response.finish_reason != "error":+ return response++ attempt = 1+ delays = self._CHAT_RETRY_DELAYSpersistent = retry_mode == "persistent"- last_response: LLMResponse | None = None+ max_attempts = None if persistent else len(delays) + 1last_error_key: str | None = Noneidentical_error_count = 0-async def _finish_retry_status(- state: Literal["recovered", "cleared"],- response: LLMResponse,+ state: Literal["recovered", "cleared", "exhausted"],+ result: LLMResponse,) -> None:- if attempt > 1 and on_retry_status:- await on_retry_status(- RetryStatusEvent(- state=state,- attempt=attempt,- max_attempts=None if persistent else len(delays) + 1,- error_kind=self.public_error_kind(response),- )- )+ if on_retry_status and (attempt > 1 or state == "exhausted"):+ await on_retry_status(RetryStatusEvent(+ state=state,+ attempt=attempt,+ max_attempts=max_attempts,+ error_kind=self.public_error_kind(result),+ ))while True:- attempt += 1- response = await call(**kw)- if response.finish_reason != "error":- await _finish_retry_status("recovered", response)- return response- last_response = responseif should_retry_guard is not None and not should_retry_guard():is_timeout = (response.error_kind or "").lower() == "timeout"if is_timeout:@@ -1632,7 +1628,6 @@ class LLMProvider(ABC):"LLM stream stalled after content was emitted; ""suppressing delta callbacks and retrying")- kw.setdefault("on_content_delta", None)kw["on_content_delta"] = Nonekw["on_thinking_delta"] = Nonekw["on_tool_call_delta"] = None@@ -1651,106 +1646,29 @@ class LLMProvider(ABC):identical_error_count = 1 if error_key else 0if not self.is_transient_response(response):- # Removing the user's images cannot repair account access.- # Do not rebuild native provider state for an unrelated failure.- if (- (response.error_kind or "").lower() in {"authentication", "oauth_auth_required"}- or response.error_status_code in {401, 403}- or self.is_arrearage_response(response)- ):- await _finish_retry_status("cleared", response)- return response- stripped = self._strip_image_content(kw["messages"])- provider_context = kw.get("provider_context")- stripped_context: ProviderCallContext | None = None- if isinstance(provider_context, ProviderCallContext):- state = provider_context.conversation_state- if state is not None and (- stripped is not None- or self._strip_image_content(state.pending_messages) is not None- or self._contains_image_content(state.payload)- ):- # Provider-owned payloads may retain earlier input_image items.- # Rebuild from the stripped public transcript for this retry.- stripped_context = ProviderCallContext(- context_window_tokens=(- provider_context.context_window_tokens- ),- session_id=provider_context.session_id,- events=provider_context.events,- response_preset=provider_context.response_preset,- response_is_fallback=provider_context.response_is_fallback,- )- if stripped is not None or stripped_context is not None:- logger.warning(- "Non-transient LLM error with image content, retrying without images"- )- retry_kw = dict(kw)- if stripped is not None:- retry_kw["messages"] = stripped- if stripped_context is not None:- retry_kw["provider_context"] = stripped_context- result = await call(**retry_kw)- # Permanently strip images from the original messages so- # subsequent iterations do not repeat the error-retry cycle.- if result.finish_reason != "error":- self._strip_image_content_inplace(original_messages)- await _finish_retry_status(- "recovered" if result.finish_reason != "error" else "cleared",- result,- )- return result- await _finish_retry_status("cleared", response)- return response-- if persistent and identical_error_count >= self._PERSISTENT_IDENTICAL_ERROR_LIMIT:- logger.warning(- "Stopping persistent retry after {} identical transient errors: {}",- identical_error_count,- (response.content or "")[:120].lower(),+ response = await self._retry_without_images(+ call, kw, original_messages, response,+ )+ await _finish_retry_status(+ "recovered" if response.finish_reason != "error" else "cleared",+ response,)- if on_retry_exhausted:- await on_retry_exhausted(- f"Persistent retry stopped after {identical_error_count} identical errors."- )- if on_retry_status:- await on_retry_status(- RetryStatusEvent(- state="exhausted",- attempt=attempt,- max_attempts=None,- error_kind=self.public_error_kind(response),- )- )return response- if not persistent and attempt > len(delays):- logger.warning(- "LLM request failed after {} attempts, giving up: {}",- attempt,- (response.content or "")[:120].lower(),- )- if on_retry_exhausted:- await on_retry_exhausted(- f"Model request failed after {attempt} attempts, giving up."- )- if on_retry_status:- await on_retry_status(- RetryStatusEvent(- state="exhausted",- attempt=attempt,- max_attempts=len(delays) + 1,- error_kind=self.public_error_kind(response),- )- )+ if (+ persistent and identical_error_count >= self._PERSISTENT_IDENTICAL_ERROR_LIMIT+ or not persistent and attempt > len(delays)+ ):breakretry_after = self._extract_retry_after_from_response(response)base_delay = delays[min(attempt - 1, len(delays) - 1)]- delay = retry_after + RETRY_AFTER_BUFFER if retry_after else base_delay+ delay = (+ retry_after + RETRY_AFTER_BUFFER+ if retry_after else random.uniform(base_delay / 2, base_delay)+ )if persistent:delay = min(delay, self._PERSISTENT_MAX_DELAY)-logger.warning("LLM transient error (attempt {}{}), retrying in {}s: {}",attempt,@@ -1763,12 +1681,83 @@ class LLMProvider(ABC):attempt=attempt,persistent=persistent,error_kind=self.public_error_kind(response),- max_attempts=None if persistent else len(delays) + 1,+ max_attempts=max_attempts,on_retry_wait=on_retry_wait,on_retry_status=on_retry_status,)+ attempt += 1+ response = await call(**kw)+ if response.finish_reason != "error":+ await _finish_retry_status("recovered", response)+ return response++ if persistent:+ logger.warning(+ "Stopping persistent retry after {} identical transient errors: {}",+ identical_error_count,+ (response.content or "")[:120].lower(),+ )+ exhausted_message = (+ f"Persistent retry stopped after {identical_error_count} identical errors."+ )+ else:+ logger.warning(+ "LLM request failed after {} attempts, giving up: {}",+ attempt,+ (response.content or "")[:120].lower(),+ )+ exhausted_message = f"Model request failed after {attempt} attempts, giving up."+ if on_retry_exhausted:+ await on_retry_exhausted(exhausted_message)+ await _finish_retry_status("exhausted", response)+ return response- return last_response if last_response is not None else await call(**kw) # pyright: ignore[reportUnnecessaryComparison]+ async def _retry_without_images(+ self,+ call: Callable[..., Awaitable[LLMResponse]],+ kw: dict[str, Any],+ original_messages: list[dict[str, Any]],+ response: LLMResponse,+ ) -> LLMResponse:+ """Try image compatibility repair once, outside transient retries."""+ # Removing the user's images cannot repair account access.+ if (+ (response.error_kind or "").lower() in {"authentication", "oauth_auth_required"}+ or response.error_status_code in {401, 403}+ or self.is_arrearage_response(response)+ ):+ return response+ stripped = self._strip_image_content(kw["messages"])+ provider_context = kw.get("provider_context")+ stripped_context: ProviderCallContext | None = None+ if isinstance(provider_context, ProviderCallContext):+ state = provider_context.conversation_state+ if state is not None and (+ stripped is not None+ or self._strip_image_content(state.pending_messages) is not None+ or self._contains_image_content(state.payload)+ ):+ # Native payloads may retain images: rebuild from the public transcript.+ stripped_context = ProviderCallContext(+ context_window_tokens=provider_context.context_window_tokens,+ session_id=provider_context.session_id,+ events=provider_context.events,+ response_preset=provider_context.response_preset,+ response_is_fallback=provider_context.response_is_fallback,+ )+ if stripped is None and stripped_context is None:+ return response+ logger.warning("Non-transient LLM error with image content, retrying without images")+ retry_kw = dict(kw)+ if stripped is not None:+ retry_kw["messages"] = stripped+ if stripped_context is not None:+ retry_kw["provider_context"] = stripped_context+ result = await call(**retry_kw)+ # Keep later iterations from repeating the same image repair.+ if result.finish_reason != "error":+ self._strip_image_content_inplace(original_messages)+ return resultasync def aclose(self) -> None:"""Close provider-owned transports after their active requests have stopped."""diff --git a/nanobot/providers/bedrock_provider.py b/nanobot/providers/bedrock_provider.pyindex d7ba30e5d..3b50dd53c 100644--- a/nanobot/providers/bedrock_provider.py+++ b/nanobot/providers/bedrock_provider.py@@ -90,7 +90,10 @@ class BedrockProvider(LLMProvider):idle_timeout_s = resolve_stream_idle_timeout_s()client_kwargs: dict[str, Any] = {- "config": Config(connect_timeout=idle_timeout_s, read_timeout=idle_timeout_s),+ "config": Config(+ connect_timeout=idle_timeout_s, read_timeout=idle_timeout_s,+ retries={"total_max_attempts": 1},+ ),}if self.region:client_kwargs["region_name"] = self.regiondiff --git a/nanobot/providers/transcription.py b/nanobot/providers/transcription.pyindex eee72dacf..8a1f26842 100644--- a/nanobot/providers/transcription.py+++ b/nanobot/providers/transcription.py@@ -350,77 +350,71 @@ async def _post_stepfun_asr_with_retry(}async with httpx.AsyncClient() as client:- for attempt in range(_MAX_RETRIES + 1):- try:- async with client.stream(- "POST", url, headers=headers, json=body, timeout=60.0- ) as resp:- if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES:- logger.warning(- "{} transcription transient HTTP {} (attempt {}/{})",- provider_label,- resp.status_code,- attempt + 1,- _MAX_RETRIES + 1,- )- await asyncio.sleep(_BACKOFF_S[attempt])+ async def request_once() -> str | None:+ # The entire stream closes before the owner decides whether to wait.+ async with client.stream(+ "POST", url, headers=headers, json=body, timeout=60.0+ ) as resp:+ resp.raise_for_status()+ async for line in resp.aiter_lines():+ if not line.startswith("data:"):+ continue+ payload_str = line[len("data:") :].strip()+ if not payload_str:continue- resp.raise_for_status()- final_text = None- async for line in resp.aiter_lines():- if not line.startswith("data:"):- continue- payload_str = line[len("data:") :].strip()- if not payload_str:- continue- try:- payload = json.loads(payload_str)- except (json.JSONDecodeError, ValueError):- continue- payload = cast(dict[str, Any], payload)- event_type = payload.get("type", "")- if event_type == "error":- msg = payload.get("message", "unknown error")- logger.error("{} ASR error: {}", provider_label, msg)- return ""- if event_type == "transcript.text.done":- final_text = payload.get("text", "")- break- if final_text is not None:- return final_text- # Stream ended without a final event — retry if attempts remain- if attempt < _MAX_RETRIES:- logger.warning(- "{} transcription: no final event (attempt {}/{})",- provider_label,- attempt + 1,- _MAX_RETRIES + 1,- )- await asyncio.sleep(_BACKOFF_S[attempt])+ try:+ payload = json.loads(payload_str)+ except (json.JSONDecodeError, ValueError):continue+ payload = cast(dict[str, Any], payload)+ event_type = payload.get("type", "")+ if event_type == "error":+ msg = payload.get("message", "unknown error")+ logger.error("{} ASR error: {}", provider_label, msg)+ return ""+ if event_type == "transcript.text.done":+ return payload.get("text", "")+ return None++ for attempt in range(_MAX_RETRIES + 1):+ try:+ final_text = await request_once()+ if final_text is not None:+ return final_text+ if attempt == _MAX_RETRIES:logger.error("{} transcription: stream ended without final text after {} attempts",provider_label,_MAX_RETRIES + 1,)return ""+ logger.warning(+ "{} transcription: no final event (attempt {}/{})",+ provider_label,+ attempt + 1,+ _MAX_RETRIES + 1,+ )except httpx.HTTPStatusError as e:- if e.response.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES:- await asyncio.sleep(_BACKOFF_S[attempt])- continue- logger.error(- "{} transcription HTTP {}{}",+ if e.response.status_code not in _RETRYABLE_STATUS or attempt == _MAX_RETRIES:+ logger.error(+ "{} transcription HTTP {}{}",+ provider_label,+ e.response.status_code,+ f" {e.response.reason_phrase}" if e.response.reason_phrase else "",+ )+ return ""+ logger.warning(+ "{} transcription transient HTTP {} (attempt {}/{})",provider_label,e.response.status_code,- f" {e.response.reason_phrase}" if e.response.reason_phrase else "",+ attempt + 1,+ _MAX_RETRIES + 1,)- return ""- except (httpx.RequestError, Exception):- if attempt < _MAX_RETRIES:- await asyncio.sleep(_BACKOFF_S[attempt])- continue- logger.exception("{} transcription request error", provider_label)- return ""+ except Exception:+ if attempt == _MAX_RETRIES:+ logger.exception("{} transcription request error", provider_label)+ return ""+ await asyncio.sleep(_BACKOFF_S[attempt])return ""diff --git a/tests/agent/test_mcp_transient_retry.py b/tests/agent/test_mcp_transient_retry.pyindex bd9b4e723..ca3f423dc 100644--- a/tests/agent/test_mcp_transient_retry.py+++ b/tests/agent/test_mcp_transient_retry.py@@ -5,10 +5,12 @@ from types import SimpleNamespacefrom unittest.mock import AsyncMock, patchimport pytest+from loguru import loggerfrom mcp import types as mcp_typesfrom mcp.shared.exceptions import McpErrorfrom mcp.types import ErrorData+from nanobot.agent.tools.context import RequestContext, request_contextfrom nanobot.agent.tools.mcp import (MCPPromptWrapper,MCPResourceWrapper,@@ -491,3 +493,135 @@ async def test_prompt_reconnects_on_connection_closed_exception():assert output == "fresh prompt"assert old_session.get_prompt.call_count == 1assert new_session.get_prompt.call_count == 1+++async def test_tool_refresh_leaves_one_transient_retry_available():+ old, fresh = AsyncMock(), AsyncMock()+ old.call_tool.side_effect = ConnectionResetError("old")+ fresh.call_tool.side_effect = [ConnectionResetError("fresh"), _make_tool_result("ok")]+ wrapper = MCPToolWrapper(old, "srv", _make_tool_def())+ reconnect = AsyncMock(return_value=MCPToolWrapper(fresh, "srv", _make_tool_def()))+ wrapper.set_reconnect_handler(reconnect)++ with patch("nanobot.agent.tools.mcp.asyncio.sleep", new_callable=AsyncMock) as sleep:+ assert await wrapper.execute(value="unchanged") == "ok"++ old.call_tool.assert_awaited_once_with("test_tool", arguments={"value": "unchanged"})+ assert fresh.call_tool.await_count == 2+ reconnect.assert_awaited_once_with("srv", wrapper.name, wrapper)+ sleep.assert_awaited_once_with(1)+++async def test_tool_later_refresh_does_not_reset_transient_budget():+ old, fresh = AsyncMock(), AsyncMock()+ old.call_tool.side_effect = [ConnectionResetError("first"), _session_terminated_error()]+ fresh.call_tool.side_effect = ConnectionResetError("last")+ wrapper = MCPToolWrapper(old, "srv", _make_tool_def())+ replacement = MCPToolWrapper(fresh, "srv", _make_tool_def())+ reconnect = AsyncMock(side_effect=[None, replacement])+ wrapper.set_reconnect_handler(reconnect)++ with patch("nanobot.agent.tools.mcp.asyncio.sleep", new_callable=AsyncMock) as sleep:+ output = await wrapper.execute()++ assert output == "(MCP tool call failed after retry: ConnectionResetError)"+ assert is_tool_error_result(output)+ assert (old.call_tool.await_count, fresh.call_tool.await_count) == (2, 1)+ assert reconnect.await_count == 2+ sleep.assert_awaited_once_with(1)++[email protected]("phase", ["refresh", "backoff"])+async def test_tool_real_cancellation_during_recovery_propagates(phase):+ entered, finished = asyncio.Event(), asyncio.Event()++ async def block(*_args):+ entered.set()+ try:+ await asyncio.Event().wait()+ finally:+ finished.set()++ session = AsyncMock()+ session.call_tool.side_effect = ConnectionResetError("gone")+ wrapper = MCPToolWrapper(session, "srv", _make_tool_def())+ if phase == "refresh":+ wrapper.set_reconnect_handler(block)+ with patch("nanobot.agent.tools.mcp.asyncio.sleep", side_effect=block):+ task = asyncio.create_task(wrapper.execute())+ try:+ await asyncio.wait_for(entered.wait(), timeout=2)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ finally:+ if not task.done():+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ assert task.cancelled() and finished.is_set()+ session.call_tool.assert_awaited_once()+++async def test_tool_reconnect_failure_is_not_a_retryable_request_failure():+ session = AsyncMock()+ session.call_tool.side_effect = ConnectionResetError("request")+ wrapper = MCPToolWrapper(session, "srv", _make_tool_def())+ failure = ConnectionResetError("refresh")+ reconnect = AsyncMock(side_effect=failure)+ wrapper.set_reconnect_handler(reconnect)+ with patch("nanobot.agent.tools.mcp.asyncio.sleep", new_callable=AsyncMock) as sleep:+ with pytest.raises(ConnectionResetError) as raised:+ await wrapper.execute()+ assert raised.value is failure+ session.call_tool.assert_awaited_once()+ reconnect.assert_awaited_once()+ sleep.assert_not_awaited()+++async def test_tool_transient_render_failure_cannot_replay_a_successful_request():+ session = AsyncMock()+ session.call_tool.return_value = _make_tool_result("ok")+ wrapper = MCPToolWrapper(session, "srv", _make_tool_def())+ reconnect = AsyncMock()+ wrapper.set_reconnect_handler(reconnect)+ with (+ patch.object(wrapper, "_render_call_result", side_effect=ConnectionResetError("Session terminated")),+ patch("nanobot.agent.tools.mcp.asyncio.sleep", new_callable=AsyncMock) as sleep,+ ):+ output = await wrapper.execute()+ assert output == "(MCP tool returned malformed content: ConnectionResetError)"+ assert is_tool_error_result(output)+ session.call_tool.assert_awaited_once()+ reconnect.assert_not_awaited()+ sleep.assert_not_awaited()++[email protected]("kind", ["tool", "prompt"])+async def test_request_error_logs_hide_content_and_prompt_retains_error_detail(kind):+ session = AsyncMock()+ if kind == "prompt":+ session.get_prompt.side_effect = McpError(ErrorData(code=-32602, message="private-request-content"))+ wrapper = MCPPromptWrapper(session, "srv", _make_prompt_def())+ old = AsyncMock()+ old.get_prompt.side_effect = _session_terminated_error()+ stale = MCPPromptWrapper(old, "srv", _make_prompt_def())+ stale.set_reconnect_handler(AsyncMock(return_value=wrapper))+ wrapper = stale+ expected = "(MCP prompt call failed: private-request-content [code -32602])"+ else:+ session.call_tool.side_effect = ValueError("private-request-content")+ wrapper = MCPToolWrapper(session, "srv", _make_tool_def())+ expected = "(MCP tool call failed: ValueError)"+ records = []+ sink = logger.add(lambda message: records.append(message.record), level="DEBUG")+ try:+ with request_context(RequestContext(channel="test", chat_id="test", log_content=False)):+ output = await wrapper.execute()+ finally:+ logger.remove(sink)+ assert output == expected+ assert is_tool_error_result(output) is (kind == "tool")+ error = next(record for record in records if record["level"].name == "ERROR")+ assert "[content hidden]" in error["message"]+ assert error["exception"] is None+ assert all("private-request-content" not in record["message"] for record in records)diff --git a/tests/agent/test_runner_fallback.py b/tests/agent/test_runner_fallback.pyindex 6eff3a41e..f162f948a 100644--- a/tests/agent/test_runner_fallback.py+++ b/tests/agent/test_runner_fallback.py@@ -4,7 +4,7 @@ from __future__ import annotationsimport asynciofrom typing import Any-from unittest.mock import AsyncMock, MagicMock, call, patch+from unittest.mock import AsyncMock, MagicMock, patchimport httpximport pytest@@ -476,7 +476,11 @@ class TestFallbackWhenPrimaryRaises:assert result.content == "fallback ok"assert result.finish_reason == "stop"factory.assert_called_once_with(_fallback("fallback-a"))- assert sleep.await_args_list == [call(1), call(2), call(4)]+ assert sleep.await_count == 3+ assert all(+ ceiling / 2 <= request.args[0] <= ceiling+ for request, ceiling in zip(sleep.await_args_list, (1, 2, 4))+ )@pytest.mark.asyncioasync def test_authentication_exception_message_is_classified(self) -> None:@@ -927,7 +931,11 @@ class TestFallbackOnPrimaryError:assert result.content == "fallback ok"assert fallback_models == ["fallback-b"]- assert sleep.await_args_list == [call(1), call(2), call(4)]+ assert sleep.await_count == 3+ assert all(+ ceiling / 2 <= request.args[0] <= ceiling+ for request, ceiling in zip(sleep.await_args_list, (1, 2, 4))+ )@pytest.mark.asyncio@pytest.mark.parametrize(("provider_name", "status", "code", "kind", "expected"), [diff --git a/tests/channels/test_channel_manager_concurrency.py b/tests/channels/test_channel_manager_concurrency.pyindex b053a890c..99ef97a81 100644--- a/tests/channels/test_channel_manager_concurrency.py+++ b/tests/channels/test_channel_manager_concurrency.py@@ -244,3 +244,132 @@ async def test_dispatcher_shutdown_cleans_active_and_waiting_destination_tasks()channel.send.assert_awaited_once()assert not manager._outbound_tasksassert not manager._outbound_tails+++async def test_all_send_slots_in_backoff_leave_healthy_chat_and_fifo_live(monkeypatch):+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (0.6,))+ manager, channel = make_manager()+ failed = set()+ delivered = []++ async def send(msg):+ if msg.chat_id != "healthy" and msg.chat_id not in failed:+ failed.add(msg.chat_id)+ raise OSError("temporary failure")+ delivered.append((msg.chat_id, msg.content))++ channel.send.side_effect = send+ channel.should_retry_send_error.return_value = True+ try:+ for i in range(manager_module._OUTBOUND_CONCURRENCY):+ await manager._queue_outbound(channel, OutboundMessage("websocket", str(i), "first"))+ await wait_until(lambda: len(failed) == manager_module._OUTBOUND_CONCURRENCY)+ await manager._queue_outbound(channel, OutboundMessage("websocket", "0", "last"))+ await manager._queue_outbound(channel, OutboundMessage("websocket", "healthy", "ok"))+ async with asyncio.timeout(0.2):+ while ("healthy", "ok") not in delivered:+ await asyncio.sleep(0)+ assert ("0", "last") not in delivered+ await wait_until(lambda: not manager._outbound_tasks)+ assert [content for chat, content in delivered if chat == "0"] == ["first", "last"]+ finally:+ await manager._cancel_outbound()+ assert not manager._outbound_tails+++async def test_scheduled_cancel_waiting_for_permit_does_not_claim(tmp_path, monkeypatch):+ from nanobot.bus.delivery import DeliveryRequest++ monkeypatch.setattr(manager_module, "_OUTBOUND_CONCURRENCY", 1)+ config = Config.model_validate({"channels": {"websocket": {"enabled": False}}})+ manager = ChannelManager(config, MessageBus(), config_path=tmp_path / "config.json")+ channel = MagicMock(spec=BaseChannel, send=AsyncMock())+ manager.channels["websocket"] = channel+ msg = OutboundMessage("websocket", "scheduled", "result",+ delivery=DeliveryRequest("run", "websocket:scheduled", None))+ manager._outbox.record(msg)+ claim = MagicMock(wraps=manager._outbox.claim)+ monkeypatch.setattr(manager._outbox, "claim", claim)+ await manager._outbound_sends.acquire()+ try:+ await manager._queue_outbound(channel, msg)+ await asyncio.sleep(0)+ await manager._cancel_outbound()+ claim.assert_not_called()+ channel.send.assert_not_awaited()+ assert [item.delivery_id for item in manager._outbox.ready()] == [msg.delivery_id]+ assert not manager._scheduled_deliveries+ finally:+ manager._outbound_sends.release()+ await manager._cancel_outbound()+++async def test_restart_notice_does_not_outwait_send_permit_deadline(tmp_path):+ from nanobot.utils.restart import RestartNotice++ config = Config.model_validate({"channels": {"websocket": {"enabled": False}}})+ manager = ChannelManager(config, MessageBus(), config_path=tmp_path / "config.json")+ channel = MagicMock(spec=BaseChannel, is_running=True)+ active = 0+ release = asyncio.Event()+ notices = []++ async def send(msg):+ nonlocal active+ if msg.chat_id == "notice":+ notices.append(msg)+ return+ active += 1+ try:+ await release.wait()+ finally:+ active -= 1++ channel.send = AsyncMock(side_effect=send)+ manager.channels["probe"] = channel+ try:+ for i in range(manager_module._OUTBOUND_CONCURRENCY):+ await manager._queue_outbound(channel, OutboundMessage("probe", str(i), "blocked"))+ await wait_until(lambda: active == manager_module._OUTBOUND_CONCURRENCY)+ await asyncio.wait_for(manager._send_restart_notice_when_started(+ RestartNotice(channel="probe", chat_id="notice", started_at_raw=""), timeout_s=0.05,+ ), 1)+ assert notices == []+ assert active == manager_module._OUTBOUND_CONCURRENCY+ finally:+ await manager._cancel_outbound()+ assert active == 0+ assert manager._outbound_sends._value == manager_module._OUTBOUND_CONCURRENCY+++async def test_restart_notice_deadline_does_not_cancel_submitted_send(tmp_path):+ from nanobot.utils.restart import RestartNotice++ config = Config.model_validate({"channels": {"websocket": {"enabled": False}}})+ manager = ChannelManager(config, MessageBus(), config_path=tmp_path / "config.json")+ started = asyncio.Event()+ release = asyncio.Event()++ async def send(msg):+ started.set()+ await release.wait()++ channel = MagicMock(spec=BaseChannel, is_running=True, send=AsyncMock(side_effect=send))+ manager.channels["probe"] = channel+ task = asyncio.create_task(manager._send_restart_notice_when_started(+ RestartNotice(channel="probe", chat_id="notice", started_at_raw=""), timeout_s=0.05,+ ))+ try:+ await asyncio.wait_for(started.wait(), 1)+ try:+ await asyncio.wait_for(asyncio.shield(task), 0.1)+ except TimeoutError:+ pass+ assert not task.done()+ release.set()+ await asyncio.wait_for(task, 1)+ channel.send.assert_awaited_once()+ assert manager._outbound_sends._value == manager_module._OUTBOUND_CONCURRENCY+ finally:+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)diff --git a/tests/channels/test_channel_plugins.py b/tests/channels/test_channel_plugins.pyindex dfe327faa..533541ad5 100644--- a/tests/channels/test_channel_plugins.py+++ b/tests/channels/test_channel_plugins.py@@ -2997,6 +2997,7 @@ async def test_send_with_retry_succeeds_first_try():mgr.bus = MessageBus()mgr.channels = {"failing": _FailingChannel(fake_config, mgr.bus)}mgr._dispatch_task = None+ mgr._outbound_sends = asyncio.Semaphore(32)msg = OutboundMessage(channel="failing", chat_id="123", content="test")await mgr._send_with_retry(mgr.channels["failing"], msg)@@ -3034,6 +3035,7 @@ async def test_send_with_retry_retries_on_failure():mgr.bus = MessageBus()mgr.channels = {"failing": _FailingChannel(fake_config, mgr.bus)}mgr._dispatch_task = None+ mgr._outbound_sends = asyncio.Semaphore(32)msg = OutboundMessage(channel="failing", chat_id="123", content="test")@@ -3075,6 +3077,7 @@ async def test_send_with_retry_no_retry_when_max_is_zero():mgr.bus = MessageBus()mgr.channels = {"failing": _FailingChannel(fake_config, mgr.bus)}mgr._dispatch_task = None+ mgr._outbound_sends = asyncio.Semaphore(32)msg = OutboundMessage(channel="failing", chat_id="123", content="test")@@ -3125,6 +3128,7 @@ async def test_send_with_retry_calls_send_delta():mgr.bus = MessageBus()mgr.channels = {"streaming": _StreamingChannel(fake_config, mgr.bus)}mgr._dispatch_task = None+ mgr._outbound_sends = asyncio.Semaphore(32)msg = outbound_message_for_event(channel="streaming",@@ -3193,6 +3197,7 @@ async def test_send_with_retry_skips_send_when_streamed():mgr.bus = MessageBus()mgr.channels = {"streamed": _StreamedChannel(fake_config, mgr.bus)}mgr._dispatch_task = None+ mgr._outbound_sends = asyncio.Semaphore(32)msg = outbound_message_for_event(channel="streamed",@@ -3309,6 +3314,7 @@ async def test_send_with_retry_propagates_cancelled_error():mgr.bus = MessageBus()mgr.channels = {"cancelling": _CancellingChannel(fake_config, mgr.bus)}mgr._dispatch_task = None+ mgr._outbound_sends = asyncio.Semaphore(32)msg = OutboundMessage(channel="cancelling", chat_id="123", content="test")@@ -3346,6 +3352,7 @@ async def test_send_with_retry_propagates_cancelled_error_during_sleep():mgr.bus = MessageBus()mgr.channels = {"failing": _FailingChannel(fake_config, mgr.bus)}mgr._dispatch_task = None+ mgr._outbound_sends = asyncio.Semaphore(32)msg = OutboundMessage(channel="failing", chat_id="123", content="test")@@ -3837,6 +3844,7 @@ async def test_restart_notice_retries_until_running_channel_accepts_delivery():mgr = ChannelManager.__new__(ChannelManager)mgr.config = fake_configmgr.bus = MessageBus()+ mgr._outbound_sends = asyncio.Semaphore(32)channel = _EventuallyDeliverableChannel(fake_config, mgr.bus)channel._running = Truemgr.channels = {"discord": channel}diff --git a/tests/channels/test_send_retry_policy.py b/tests/channels/test_send_retry_policy.pynew file mode 100644index 000000000..9847eaf82--- /dev/null+++ b/tests/channels/test_send_retry_policy.py@@ -0,0 +1,127 @@+"""Reachable manager/adapter ownership contracts for outbound retries."""++import asyncio+import random+from types import SimpleNamespace+from unittest.mock import AsyncMock, Mock++import pytest++# Telegram is an optional channel dependency, as in its adapter-owned suite.+pytest.importorskip("telegram")+from telegram.error import NetworkError, RetryAfter, TimedOut++from nanobot.bus.delivery import ChannelNotReadyError+from nanobot.bus.events import OutboundMessage+from nanobot.bus.queue import MessageBus+from nanobot.channels import manager as manager_module+from nanobot.channels.manager import ChannelManager+from nanobot.channels.telegram import runtime as telegram_module+from nanobot.channels.telegram.runtime import (+ TELEGRAM_MAX_MESSAGE_LEN,+ TelegramChannel,+ _split_telegram_markdown,+)+from nanobot.config.schema import Config+++def make_manager(tmp_path):+ config = Config.model_validate({"channels": {"websocket": {"enabled": False}}})+ return ChannelManager(config, MessageBus(), config_path=tmp_path / "config.json")+++def telegram_channel(manager):+ channel = TelegramChannel({"token": "test", "rich_messages": False}, manager.bus)+ channel._app = SimpleNamespace(bot=SimpleNamespace(send_message=AsyncMock()))+ channel._app_ready.set()+ return channel+++async def test_telegram_exhausted_second_chunk_never_replays_successful_prefix(tmp_path, monkeypatch):+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (0,))+ monkeypatch.setattr(telegram_module, "_SEND_RETRY_BASE_DELAY", 0)+ manager = make_manager(tmp_path)+ channel = telegram_channel(manager)+ text = "a" * TELEGRAM_MAX_MESSAGE_LEN + "\n" + "b" * 20+ chunks = _split_telegram_markdown(text, TELEGRAM_MAX_MESSAGE_LEN)+ assert len(chunks) == 2+ submitted = []++ async def send_message(**kwargs):+ submitted.append(kwargs["text"])+ if kwargs["text"] != chunks[0]:+ raise TimedOut("second part exhausted")+ return SimpleNamespace(message_id=1)++ channel._app.bot.send_message.side_effect = send_message+ await manager._send_with_retry(channel, OutboundMessage("telegram", "1", text))+ assert submitted == [chunks[0], chunks[1], chunks[1], chunks[1]]++[email protected](("error", "requests"), [+ (TimedOut("timeout"), 3),+ (RetryAfter(0), 3),+ (NetworkError("ambiguous transport error"), 1),+ (RuntimeError("unknown send result"), 1),+])+async def test_telegram_manager_does_not_multiply_part_retry_budget(tmp_path, monkeypatch, error, requests):+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (0,))+ monkeypatch.setattr(telegram_module, "_SEND_RETRY_BASE_DELAY", 0)+ manager = make_manager(tmp_path)+ channel = telegram_channel(manager)+ channel._app.bot.send_message.side_effect = error+ await manager._send_with_retry(channel, OutboundMessage("telegram", "1", "result"))+ assert channel._app.bot.send_message.await_count == requests+++async def test_telegram_manager_retries_only_pre_submit_not_ready(tmp_path, monkeypatch):+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (0,))+ manager = make_manager(tmp_path)+ channel = telegram_channel(manager)+ channel._wait_for_app = AsyncMock(side_effect=[None, channel._app])+ await manager._send_with_retry(channel, OutboundMessage("telegram", "1", "result"))+ assert channel._wait_for_app.await_count == 2+ channel._app.bot.send_message.assert_awaited_once()+++async def test_manager_equal_jitter_uses_capped_retry_delays(tmp_path, monkeypatch):+ manager = make_manager(tmp_path)+ manager.config.channels.send_max_retries = 6+ channel = SimpleNamespace(send=AsyncMock(side_effect=OSError("offline")),+ should_retry_send_error=Mock(return_value=True))+ uniform = Mock(side_effect=lambda lower, upper: (lower + upper) / 2)+ monkeypatch.setattr(random, "uniform", uniform)+ sleep = AsyncMock()+ monkeypatch.setattr(asyncio, "sleep", sleep)+ await manager._send_with_retry(channel, OutboundMessage("test", "1", "result"))+ assert [call.args for call in uniform.call_args_list] == [(0.5, 1), (1.0, 2), (2.0, 4), (2.0, 4), (2.0, 4)]+ assert [call.args[0] for call in sleep.await_args_list] == [0.75, 1.5, 3, 3, 3]+++async def test_manager_deadline_clamps_jitter_and_keeps_pre_submit_retry(tmp_path, monkeypatch):+ manager = make_manager(tmp_path)+ manager.config.channels.send_max_retries = 1+ channel = SimpleNamespace(send=AsyncMock(side_effect=[ChannelNotReadyError("starting"), None]),+ should_retry_send_error=Mock(return_value=True))+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (4,))+ monkeypatch.setattr(random, "uniform", lambda lower, upper: upper)+ sleep = AsyncMock()+ monkeypatch.setattr(asyncio, "sleep", sleep)+ await manager._send_with_retry(channel, OutboundMessage("test", "1", "result"),+ deadline=asyncio.get_running_loop().time() + 2)+ assert channel.send.await_count == 2+ assert 0 < sleep.await_args.args[0] <= 2+++async def test_telegram_restart_notice_deadline_never_replays_unknown_result(tmp_path):+ from nanobot.utils.restart import RestartNotice++ manager = make_manager(tmp_path)+ channel = telegram_channel(manager)+ channel._running = True+ channel._app.bot.send_message.side_effect = NetworkError("unknown delivery")+ manager.channels["telegram"] = channel+ await manager._send_restart_notice_when_started(+ RestartNotice(channel="telegram", chat_id="1", started_at_raw=""), timeout_s=0.1,+ )+ channel._app.bot.send_message.assert_awaited_once()diff --git a/tests/providers/test_openai_codex_provider.py b/tests/providers/test_openai_codex_provider.pyindex 7ad250ea6..636eeaac5 100644--- a/tests/providers/test_openai_codex_provider.py+++ b/tests/providers/test_openai_codex_provider.py@@ -698,7 +698,8 @@ async def test_codex_retry_uses_structured_transient_error_metadata(monkeypatch,assert response.content == "ok"assert calls == 2- assert delays == [1]+ assert len(delays) == 1+ assert 0.5 <= delays[0] <= 1.0@pytest.mark.asynciodiff --git a/tests/providers/test_provider_retry.py b/tests/providers/test_provider_retry.pyindex 8799793ec..3d939cd15 100644--- a/tests/providers/test_provider_retry.py+++ b/tests/providers/test_provider_retry.py@@ -1,5 +1,6 @@import asyncioimport copy+import randomimport pytest@@ -14,6 +15,11 @@ from nanobot.providers.base import ()+def assert_local_backoff(delays: list[float], ceilings: tuple[float, ...]) -> None:+ assert len(delays) == len(ceilings)+ assert all(ceiling / 2 <= delay <= ceiling for delay, ceiling in zip(delays, ceilings))++class ScriptedProvider(LLMProvider):def __init__(self, responses):super().__init__(provider_name="scripted")@@ -64,9 +70,9 @@ async def test_chat_with_retry_retries_transient_error_then_succeeds(monkeypatchLLMResponse(content="429 rate limit", finish_reason="error"),LLMResponse(content="ok"),])- delays: list[int] = []+ delays: list[float] = []- async def _fake_sleep(delay: int) -> None:+ async def _fake_sleep(delay: float) -> None:delays.append(delay)monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)@@ -76,7 +82,7 @@ async def test_chat_with_retry_retries_transient_error_then_succeeds(monkeypatchassert response.finish_reason == "stop"assert response.content == "ok"assert provider.calls == 2- assert delays == [1]+ assert_local_backoff(delays, (1,))@pytest.mark.asyncio@@ -156,9 +162,9 @@ async def test_chat_with_retry_does_not_retry_non_transient_error(monkeypatch) -provider = ScriptedProvider([LLMResponse(content="401 unauthorized", finish_reason="error"),])- delays: list[int] = []+ delays: list[float] = []- async def _fake_sleep(delay: int) -> None:+ async def _fake_sleep(delay: float) -> None:delays.append(delay)monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)@@ -171,25 +177,37 @@ async def test_chat_with_retry_does_not_retry_non_transient_error(monkeypatch) -@pytest.mark.asyncio-async def test_chat_with_retry_returns_final_error_after_retries(monkeypatch) -> None:[email protected]("fraction", [0.0, 1.0])+async def test_chat_with_retry_returns_final_error_after_retries(monkeypatch, fraction) -> None:provider = ScriptedProvider([LLMResponse(content="429 rate limit a", finish_reason="error"),LLMResponse(content="429 rate limit b", finish_reason="error"),LLMResponse(content="429 rate limit c", finish_reason="error"),LLMResponse(content="503 final server error", finish_reason="error"),])- delays: list[int] = []+ delays: list[float] = []+ statuses: list[RetryStatusEvent] = []+ monkeypatch.setattr(random, "uniform", lambda low, high: low + (high - low) * fraction)+ monkeypatch.setattr("nanobot.providers.base.time.time", lambda: 1000.0)- async def _fake_sleep(delay: int) -> None:+ async def _fake_sleep(delay: float) -> None:delays.append(delay)- monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)+ async def status(event: RetryStatusEvent) -> None:+ statuses.append(event)- response = await provider.chat_with_retry(messages=[{"role": "user", "content": "hello"}])+ monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)+ response = await provider.chat_with_retry(+ messages=[{"role": "user", "content": "hello"}], on_retry_status=status,+ )assert response.content == "503 final server error"assert provider.calls == 4- assert delays == [1, 2, 4]+ assert delays == [base * (0.5 + fraction / 2) for base in (1, 2, 4)]+ assert [event.next_retry_at for event in statuses if event.state == "waiting"] == [+ 1000.0 + delay for delay in delays+ ]+ assert statuses[-1].state == "exhausted"@pytest.mark.asyncio@@ -207,7 +225,7 @@ async def test_chat_with_retry_emits_terminal_progress_when_standard_retries_exhprogress: list[str] = []statuses: list[RetryStatusEvent] = []- async def _fake_sleep(delay: int) -> None:+ async def _fake_sleep(delay: float) -> None:return Noneasync def _progress(msg: str) -> None:@@ -245,7 +263,7 @@ async def test_chat_with_retry_routes_terminal_progress_to_explicit_callback(monretry_progress: list[str] = []terminal_progress: list[str] = []- async def _fake_sleep(delay: int) -> None:+ async def _fake_sleep(delay: float) -> None:return Noneasync def _retry_progress(msg: str) -> None:@@ -284,9 +302,9 @@ async def test_chat_stream_with_retry_does_not_retry_after_emitting_content(monkLLMResponse(content="ok"),])deltas: list[str] = []- delays: list[int] = []+ delays: list[float] = []- async def _fake_sleep(delay: int) -> None:+ async def _fake_sleep(delay: float) -> None:delays.append(delay)async def _on_delta(delta: str) -> None:@@ -318,9 +336,9 @@ async def test_chat_stream_with_retry_retries_timeout_after_emitting_content(monLLMResponse(content="full retry response"),])deltas: list[str] = []- delays: list[int] = []+ delays: list[float] = []- async def _fake_sleep(delay: int) -> None:+ async def _fake_sleep(delay: float) -> None:delays.append(delay)async def _on_delta(delta: str) -> None:@@ -337,7 +355,7 @@ async def test_chat_stream_with_retry_retries_timeout_after_emitting_content(monassert response.finish_reason == "stop"assert provider.calls == 2assert deltas == ["partial"]- assert delays == [1]+ assert_local_backoff(delays, (1,))assert provider.last_kwargs.get("on_content_delta") is None@@ -356,9 +374,9 @@ async def test_chat_stream_with_retry_retries_timeout_in_new_stream_segment(provider = ScriptedProvider([first, second])deltas: list[str] = []recoveries: list[str] = []- delays: list[int] = []+ delays: list[float] = []- async def _fake_sleep(delay: int) -> None:+ async def _fake_sleep(delay: float) -> None:delays.append(delay)async def _on_delta(delta: str) -> None:@@ -380,7 +398,7 @@ async def test_chat_stream_with_retry_retries_timeout_in_new_stream_segment(assert provider.calls == 2assert deltas == ["partial", "full retry response"]assert recoveries == ["recover"]- assert delays == [1]+ assert_local_backoff(delays, (1,))assert provider.last_kwargs.get("on_content_delta") is not None@@ -633,7 +651,8 @@ async def test_image_fallback_without_meta_uses_default_placeholder() -> None:@pytest.mark.asyncio-async def test_chat_with_retry_uses_retry_after_and_emits_wait_progress(monkeypatch) -> None:[email protected]("retry_mode", ["standard", "persistent"])+async def test_chat_with_retry_uses_retry_after_and_emits_wait_progress(monkeypatch, retry_mode) -> None:provider = ScriptedProvider([LLMResponse(content="429 rate limit, retry after 7s", finish_reason="error"),LLMResponse(content="ok"),@@ -641,17 +660,21 @@ async def test_chat_with_retry_uses_retry_after_and_emits_wait_progress(monkeypadelays: list[float] = []progress: list[str] = []+ def unexpected_draw(*args):+ pytest.fail("A provider-directed Retry-After must not be jittered")+async def _fake_sleep(delay: float) -> None:delays.append(delay)async def _progress(msg: str) -> None:progress.append(msg)+ monkeypatch.setattr(random, "uniform", unexpected_draw)monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)response = await provider.chat_with_retry(messages=[{"role": "user", "content": "hello"}],- on_retry_wait=_progress,+ on_retry_wait=_progress, retry_mode=retry_mode,)assert response.content == "ok"@@ -744,7 +767,7 @@ async def test_chat_with_retry_retries_structured_status_code_without_keyword(moassert response.content == "ok"assert provider.calls == 2- assert delays == [1]+ assert_local_backoff(delays, (1,))@pytest.mark.asyncio@@ -821,7 +844,7 @@ async def test_chat_with_retry_retries_structured_timeout_kind(monkeypatch) -> Nassert response.content == "ok"assert provider.calls == 2- assert delays == [1]+ assert_local_backoff(delays, (1,))@pytest.mark.asyncio@@ -891,7 +914,7 @@ async def test_persistent_retry_aborts_after_ten_identical_transient_errors(monkassert response.finish_reason == "error"assert response.content == "429 rate limit"assert provider.calls == 10- assert delays == [1, 2, 4, 4, 4, 4, 4, 4, 4]+ assert_local_backoff(delays, (1, 2, 4, 4, 4, 4, 4, 4, 4))@pytest.mark.asyncio@@ -963,7 +986,7 @@ async def test_chat_with_retry_retries_zhipu_1302_rate_limit(monkeypatch) -> Nonassert response.content == "ok"assert provider.calls == 2- assert delays == [1]+ assert_local_backoff(delays, (1,))@pytest.mark.asyncio@@ -989,7 +1012,7 @@ async def test_chat_with_retry_retries_zhipu_1302_with_429_status(monkeypatch) -assert response.content == "ok"assert provider.calls == 2- assert delays == [1]+ assert_local_backoff(delays, (1,))@pytest.mark.asyncio@@ -1006,3 +1029,60 @@ async def test_chat_stream_with_retry_normalizes_explicit_none_max_tokens() -> Nassert response.content == "ok"assert provider.last_kwargs["max_tokens"] == 4096assert provider.last_kwargs["temperature"] == 0.7+++async def test_concurrent_requests_keep_retry_status_and_attempts_local(monkeypatch) -> None:+ ready = asyncio.Event()+ calls = {"a": 0, "b": 0}+ statuses = {"a": [], "b": []}++ class ConcurrentProvider(ScriptedProvider):+ async def chat(self, messages, **kwargs):+ key = messages[-1]["content"]+ calls[key] += 1+ if calls[key] == 1:+ if all(calls.values()):+ ready.set()+ await ready.wait()+ return LLMResponse(content="connection failed", finish_reason="error")+ return LLMResponse(content=key)++ provider = ConcurrentProvider([])+ monkeypatch.setattr(provider, "_CHAT_RETRY_DELAYS", (0.1,))++ async def request(key):+ async def status(event):+ statuses[key].append((event.state, event.attempt, event.max_attempts))++ return await provider.chat_with_retry(+ messages=[{"role": "user", "content": key}], on_retry_status=status,+ )++ results = await asyncio.wait_for(asyncio.gather(request("a"), request("b")), 2)+ assert [result.content for result in results] == ["a", "b"]+ assert calls == {"a": 2, "b": 2}+ assert statuses == {key: [("waiting", 1, 2), ("recovered", 2, 2)] for key in calls}+++async def test_cancellation_interrupts_retry_wait_without_another_call() -> None:+ provider = ScriptedProvider([LLMResponse(+ content="rate limited", finish_reason="error", error_status_code=429, error_retry_after_s=120,+ )])+ waiting = asyncio.Event()+ statuses = []++ async def status(event):+ statuses.append((event.state, event.attempt))+ waiting.set()++ task = asyncio.create_task(provider.chat_with_retry(messages=[], on_retry_status=status))+ try:+ await asyncio.wait_for(waiting.wait(), 2)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ finally:+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ assert provider.calls == 1+ assert statuses == [("waiting", 1)]diff --git a/tests/providers/test_provider_sdk_retry_defaults.py b/tests/providers/test_provider_sdk_retry_defaults.pyindex bf4f10bda..b5fb86a19 100644--- a/tests/providers/test_provider_sdk_retry_defaults.py+++ b/tests/providers/test_provider_sdk_retry_defaults.py@@ -1,7 +1,10 @@from unittest.mock import patch+import pytest+from nanobot.providers.anthropic_provider import AnthropicProviderfrom nanobot.providers.azure_openai_provider import AzureOpenAIProvider+from nanobot.providers.bedrock_provider import BedrockProviderfrom nanobot.providers.openai_compat_provider import OpenAICompatProvider@@ -44,3 +47,13 @@ def test_azure_openai_disables_sdk_retries_by_default() -> None:kwargs = mock_client.call_args.kwargsassert kwargs["max_retries"] == 0+++def test_bedrock_disables_sdk_retries_even_with_environment_attempts(monkeypatch) -> None:+ pytest.importorskip("boto3")+ monkeypatch.setenv("AWS_MAX_ATTEMPTS", "10")+ with patch("boto3.Session") as session:+ BedrockProvider(api_base="http://127.0.0.1:1", region="us-east-1")++ config = session.return_value.client.call_args.kwargs["config"]+ assert config.retries["total_max_attempts"] == 1diff --git a/tests/providers/test_stepfun_asr.py b/tests/providers/test_stepfun_asr.pyindex 4074f0a7e..867cb26f6 100644--- a/tests/providers/test_stepfun_asr.py+++ b/tests/providers/test_stepfun_asr.py@@ -2,6 +2,8 @@from __future__ import annotations+import asyncio+import base64import jsonfrom pathlib import Pathfrom typing import Any@@ -132,11 +134,14 @@ async def test_sse_error_event(audio_file: Path) -> None:provider = StepFunTranscriptionProvider(api_key="sk-test")stream_cm = _make_stream_cm(200, lines)+ sleep = AsyncMock()- with patch("httpx.AsyncClient.stream", stream_cm):+ with patch("httpx.AsyncClient.stream", stream_cm), patch("asyncio.sleep", sleep):result = await provider.transcribe(audio_file)assert result == ""+ assert stream_cm.call_count == 1+ sleep.assert_not_awaited()@pytest.mark.asyncio@@ -186,36 +191,78 @@ async def test_sse_malformed_json_skipped(audio_file: Path) -> None:@pytest.mark.asyncio-async def test_retries_on_503_then_succeeds(audio_file: Path) -> None:- """Transient 503 is retried, then a successful SSE stream yields text."""- success_lines = [- f"data: {json.dumps({'type': 'transcript.text.done', 'session_id': 's1', 'text': 'ok'})}",- ]- # First call: 503 (FailingResponse), second call: success (FakeResponse with lines)- stream_cm = _make_stream_cm_sequence([503, success_lines])-- provider = StepFunTranscriptionProvider(api_key="sk-test")- with patch("httpx.AsyncClient.stream", stream_cm), patch(- "asyncio.sleep", AsyncMock()- ):- result = await provider.transcribe(audio_file)-- assert result == "ok"+ "scenario,expected",+ [+ ([(503, [], None), (200, [], None), (200, ["data: []"], None),+ (200, ['data: {"type":"transcript.text.done","text":"ok"}'], None)], "ok"),+ ([(200, [], httpx.ReadError("lost")),+ (200, ['data: {"type":"transcript.text.done","text":"ok"}'], None)], "ok"),+ ([(200, [], None)] * 4, ""),+ ([(200, ["data: []"], None)] * 4, ""),+ ],+ ids=["mixed-budget", "read-error", "missing-final-exhaustion", "parser-exhaustion"],+)+async def test_sse_closes_each_attempt_before_backoff(monkeypatch, audio_file, scenario, expected):+ original_audio = audio_file.read_bytes()+ remaining = iter(scenario)+ streams, bodies, waits = [], [], []++ class Stream(httpx.AsyncByteStream):+ closed = False++ def __init__(self, lines, error):+ self.lines, self.error = lines, error++ async def __aiter__(self):+ for line in self.lines:+ yield (line + "\n").encode()+ if self.error is not None:+ raise self.error++ async def aclose(self):+ self.closed = True++ def handle(request):+ bodies.append(json.loads(request.content))+ status, lines, error = next(remaining)+ stream = Stream(lines, error)+ streams.append(stream)+ return httpx.Response(status, stream=stream)++ client = httpx.AsyncClient(transport=httpx.MockTransport(handle))++ async def sleep(delay):+ assert streams[-1].closed+ waits.append(delay)+ audio_file.write_bytes(b"changed-after-request")++ monkeypatch.setattr("nanobot.providers.transcription.httpx.AsyncClient", lambda: client)+ monkeypatch.setattr("nanobot.providers.transcription.asyncio.sleep", sleep)+ result = await StepFunTranscriptionProvider(api_key="sk-test", language="zh").transcribe(audio_file)+ assert result == expected+ assert len(bodies) == len(scenario)+ assert bodies == [bodies[0]] * len(scenario)+ assert base64.b64decode(bodies[0]["audio"]["data"]) == original_audio+ assert bodies[0]["audio"]["input"]["transcription"]["language"] == "zh"+ assert waits == [1.0, 2.0, 4.0][:len(scenario) - 1]+ assert all(stream.closed for stream in streams)+ assert client.is_closed@pytest.mark.asyncioasync def test_gives_up_after_max_retries(audio_file: Path) -> None:"""Persistent 503 returns "" after all retries exhausted."""- attempts: list[list[str] | int] = [503, 503, 503, 503] # 4 failing HTTP responses- stream_cm = _make_stream_cm_sequence(attempts)+ stream_cm = _make_stream_cm(503, [])provider = StepFunTranscriptionProvider(api_key="sk-test")- with patch("httpx.AsyncClient.stream", stream_cm), patch(- "asyncio.sleep", AsyncMock()- ):+ sleep = AsyncMock()+ with patch("httpx.AsyncClient.stream", stream_cm), patch("asyncio.sleep", sleep):result = await provider.transcribe(audio_file)assert result == ""+ assert stream_cm.call_count == 4+ assert [call.args[0] for call in sleep.await_args_list] == [1.0, 2.0, 4.0]@pytest.mark.asyncio@@ -229,18 +276,19 @@ async def test_sse_empty_text_done_returns_empty(audio_file: Path) -> None:provider = StepFunTranscriptionProvider(api_key="sk-test")stream_cm = _make_stream_cm(200, lines)- with patch("httpx.AsyncClient.stream", stream_cm), patch(- "asyncio.sleep", AsyncMock()- ):+ sleep = AsyncMock()+ with patch("httpx.AsyncClient.stream", stream_cm), patch("asyncio.sleep", sleep):result = await provider.transcribe(audio_file)assert result == ""+ assert stream_cm.call_count == 1+ sleep.assert_not_awaited()@pytest.mark.asyncio-async def test_401_returns_empty_without_retry(audio_file: Path) -> None:- """401 is not retryable; bad credentials should fail immediately."""- stream_cm = _make_stream_cm(401, [])[email protected]("status", [401, 501])+async def test_non_retryable_status_returns_empty(audio_file: Path, status: int) -> None:+ stream_cm = _make_stream_cm(status, [])sleep = AsyncMock()provider = StepFunTranscriptionProvider(api_key="sk-test")@@ -258,32 +306,9 @@ async def test_retries_on_connect_error(audio_file: Path) -> None:success_lines = [f"data: {json.dumps({'type': 'transcript.text.done', 'session_id': 's1', 'text': 'ok'})}",]- call_count = [0]-- class FakeResponse:- """Serves as both the async context manager returned by stream()- and the response object bound in `async with ... as resp`."""- status_code = 200- reason_phrase = "OK"-- async def __aenter__(self) -> "FakeResponse":- return self-- async def __aexit__(self, *exc: object) -> None:- pass-- async def aiter_lines(self) -> Any:- for line in success_lines:- yield line-- def raise_for_status(self) -> None:- pass-- def fake_stream(method: str, url: str, *args: object, **kwargs: object) -> FakeResponse:- call_count[0] += 1- if call_count[0] == 1:- raise httpx.ConnectError("boom")- return FakeResponse()+ fake_stream = MagicMock(side_effect=[+ httpx.ConnectError("boom"), _make_stream_cm(200, success_lines).return_value,+ ])provider = StepFunTranscriptionProvider(api_key="sk-test")with patch("httpx.AsyncClient.stream", fake_stream), patch(@@ -292,7 +317,82 @@ async def test_retries_on_connect_error(audio_file: Path) -> None:result = await provider.transcribe(audio_file)assert result == "ok"- assert call_count[0] == 2+ assert fake_stream.call_count == 2++[email protected]("cancel_backoff", [False, True], ids=["recover", "cancel-backoff"])+async def test_unread_503_socket_closes_before_backoff(monkeypatch, audio_file, cancel_backoff):+ first_closed, entered, blocked = asyncio.Event(), asyncio.Event(), asyncio.Event()+ bodies, responses, waits = [], [], []+ handlers = set()++ async def handle(reader, writer):+ handler = asyncio.current_task()+ handlers.add(handler)+ try:+ head = await reader.readuntil(b"\r\n\r\n")+ headers = dict(line.split(b":", 1) for line in head.split(b"\r\n")[1:] if b":" in line)+ bodies.append(json.loads(await reader.readexactly(int(headers[b"Content-Length"]))))+ first = len(bodies) == 1+ if first:+ # The body never finishes: the retry must close it without reading EOF.+ writer.write(b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 1000000\r\n\r\nunread")+ else:+ payload = b'data: {"type":"transcript.text.done","text":"socket-ok"}\n\n'+ writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: "+ + str(len(payload)).encode() + b"\r\n\r\n" + payload)+ await writer.drain()+ await reader.read()+ if first:+ first_closed.set()+ finally:+ writer.close()+ await writer.wait_closed()+ handlers.discard(handler)++ async def response_received(response):+ responses.append(response)++ async def sleep(delay):+ waits.append(delay)+ assert responses[-1].is_closed+ await asyncio.wait_for(first_closed.wait(), timeout=2)+ entered.set()+ if cancel_backoff:+ await blocked.wait()++ server = await asyncio.start_server(handle, "127.0.0.1", 0)+ port = server.sockets[0].getsockname()[1]+ client = httpx.AsyncClient(trust_env=False, event_hooks={"response": [response_received]})+ monkeypatch.setattr("nanobot.providers.transcription.httpx.AsyncClient", lambda: client)+ monkeypatch.setattr("nanobot.providers.transcription.asyncio.sleep", sleep)+ task = asyncio.create_task(StepFunTranscriptionProvider(+ api_key="local", api_base=f"http://127.0.0.1:{port}",+ ).transcribe(audio_file))+ try:+ await asyncio.wait_for(entered.wait(), timeout=5)+ if cancel_backoff:+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ assert len(bodies) == 1+ else:+ assert await asyncio.wait_for(task, timeout=5) == "socket-ok"+ assert len(bodies) == 2+ assert bodies[0] == bodies[1]+ assert waits == [1.0]+ assert client.is_closed+ finally:+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ await client.aclose()+ server.close()+ await server.wait_closed()+ pending = list(handlers)+ for handler in pending:+ handler.cancel()+ await asyncio.wait_for(asyncio.gather(*pending, return_exceptions=True), timeout=5)# ---------------------------------------------------------------------------@@ -362,66 +462,3 @@ def _make_stream_cm(status: int, lines: list[str]) -> MagicMock:cm = MagicMock()cm.return_value = FakeResponse()return cm---def _make_stream_cm_sequence(statuses: list[str | int]) -> MagicMock:- """Build a stream mock that fails with HTTP status ints, then succeeds with SSE lines.-- Entries in *statuses* that are ints produce a stream that raises HTTPStatusError- after `raise_for_status()`. The final entry (a list of SSE lines) succeeds.- """- remaining = list(statuses)-- class FakeResponse:- def __init__(self, lines: list[str]) -> None:- self._lines = lines- self.status_code = 200- self.reason_phrase = "OK"-- async def __aenter__(self) -> "FakeResponse":- return self-- async def __aexit__(self, *exc: object) -> None:- pass-- async def aiter_lines(self) -> Any:- for line in self._lines:- yield line-- def raise_for_status(self) -> None:- pass-- class FailingResponse:- def __init__(self, status: int) -> None:- self.status_code = status- self.reason_phrase = "Error"-- async def __aenter__(self) -> "FailingResponse":- return self-- async def __aexit__(self, *exc: object) -> None:- pass-- async def aiter_lines(self) -> Any:- yield ""- return-- def raise_for_status(self) -> None:- raise httpx.HTTPStatusError(- f"HTTP {self.status_code}",- request=httpx.Request("POST", "https://example.test"),- response=httpx.Response(self.status_code),- )-- call_count = [0]-- def _next(method: str, url: str, **kwargs: object) -> Any:- idx = min(call_count[0], len(remaining) - 1)- entry = remaining[idx]- call_count[0] += 1- if isinstance(entry, int):- return FailingResponse(entry)- return FakeResponse(entry)-- cm = MagicMock(side_effect=_next)- return cmdiff --git a/tests/providers/test_transcription.py b/tests/providers/test_transcription.pyindex 99ae1258a..d1ad4d744 100644--- a/tests/providers/test_transcription.py+++ b/tests/providers/test_transcription.py@@ -2,6 +2,7 @@from __future__ import annotations+import asyncioimport base64import osfrom pathlib import Path@@ -26,6 +27,7 @@ from nanobot.providers.transcription import (GroqTranscriptionProvider,OpenAITranscriptionProvider,OpenRouterTranscriptionProvider,+ StepFunTranscriptionProvider,XiaomiMiMoTranscriptionProvider,_audio_format,_resolve_chat_completions_url,@@ -419,40 +421,30 @@ def test_resolver_keeps_enabled_and_limits_on_effective_config() -> None:@pytest.mark.asyncio-async def test_openai_does_not_retry_on_auth_error(audio_file: Path) -> None:- """401 is the user's misconfiguration — retrying wastes time and rate-limit quota."""[email protected]("outcome", [_response(401), _response(501), httpx.LocalProtocolError("bad")])+async def test_openai_does_not_retry_terminal_errors(audio_file: Path, outcome) -> None:provider = OpenAITranscriptionProvider(api_key="sk-test")- post = AsyncMock(return_value=_response(401, {"error": {"message": "bad key"}}))- with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", AsyncMock()):+ post = AsyncMock(side_effect=[outcome])+ sleep = AsyncMock()+ with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", sleep):result = await provider.transcribe(audio_file)assert result == ""assert post.await_count == 1+ sleep.assert_not_awaited()@pytest.mark.asyncio-async def test_openai_gives_up_after_max_attempts(audio_file: Path) -> None:- """Persistent 503 returns "" after the final retry — never hangs."""[email protected]("outcome", [_response(503), httpx.ConnectError("offline")])+async def test_openai_gives_up_after_max_attempts(audio_file: Path, outcome) -> None:provider = OpenAITranscriptionProvider(api_key="sk-test")- post = AsyncMock(return_value=_response(503))+ post = AsyncMock(side_effect=[outcome] * 4)sleep = AsyncMock()with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", sleep):result = await provider.transcribe(audio_file)assert result == ""# 4 attempts total (initial + 3 retries) with 3 sleeps between them.assert post.await_count == 4- assert sleep.await_count == 3---async def test_openai_backoff_grows_exponentially(audio_file: Path) -> None:- """Verify the backoff schedule is exponential (1s, 2s, 4s)."""- provider = OpenAITranscriptionProvider(api_key="sk-test")- post = AsyncMock(return_value=_response(503))- sleep = AsyncMock()- with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", sleep):- await provider.transcribe(audio_file)- delays = [call.args[0] for call in sleep.await_args_list]- assert delays == [1.0, 2.0, 4.0]+ assert [call.args[0] for call in sleep.await_args_list] == [1.0, 2.0, 4.0]# ---------------------------------------------------------------------------@@ -590,16 +582,33 @@ async def test_provider_forwards_file_mime_type(tmp_path: Path) -> None:@pytest.mark.asyncio-async def test_language_survives_retry(audio_file: Path) -> None:- """Regression: language must be present on every retry attempt, not just the first."""- provider = OpenAITranscriptionProvider(api_key="sk-test", language="ja")- post = AsyncMock(side_effect=[_response(503), _response(200, {"text": "konnichiwa"})])- with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", AsyncMock()):- result = await provider.transcribe(audio_file)- assert result == "konnichiwa"- assert post.await_count == 2[email protected]("provider_cls", [OpenAITranscriptionProvider, OpenRouterTranscriptionProvider])+async def test_json_retry_shares_budget_and_rebuilds_cached_audio(audio_file: Path, provider_cls):+ original_audio = audio_file.read_bytes()+ provider = provider_cls(api_key="sk-test", model="contract-model", language="ja")+ post = AsyncMock(side_effect=[+ httpx.ReadError("lost"), _response(503), httpx.WriteError("lost"),+ _response(200, {"text": "konnichiwa"}),+ ])+ sleep = AsyncMock(side_effect=lambda _: audio_file.write_bytes(b"changed-after-request"))+ with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", sleep):+ assert await provider.transcribe(audio_file) == "konnichiwa"+ assert post.await_count == 4+ assert [call.args[0] for call in sleep.await_args_list] == [1.0, 2.0, 4.0]+ field = "files" if provider_cls is OpenAITranscriptionProvider else "json"+ bodies = [call.kwargs[field] for call in post.await_args_list]+ assert len({id(body) for body in bodies}) == 4+ expected = (+ {"file": (audio_file.name, original_audio, "audio/ogg"),+ "model": (None, "contract-model"), "language": (None, "ja")}+ if field == "files" else+ {"model": "contract-model", "language": "ja",+ "input_audio": {"data": base64.b64encode(original_audio).decode(), "format": "ogg"}}+ )+ assert bodies == [expected] * 4for call in post.await_args_list:- assert call.kwargs["files"]["language"] == (None, "ja")+ assert call.kwargs["headers"]["Authorization"] == "Bearer sk-test"+ assert call.kwargs["timeout"] == 60.0# ---------------------------------------------------------------------------@@ -713,16 +722,6 @@ async def test_openrouter_omits_language_when_unset(audio_file: Path) -> None:assert "language" not in post.await_args_list[0].kwargs["json"]-async def test_openrouter_shares_retry_contract(audio_file: Path) -> None:- """OpenRouter goes through the same retry helper: 503 retried, then 200."""- provider = OpenRouterTranscriptionProvider(api_key="k", model="openai/whisper-1")- post = AsyncMock(side_effect=[_response(503), _response(200, {"text": "recovered"})])- with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", AsyncMock()):- assert await provider.transcribe(audio_file) == "recovered"- assert post.await_count == 2--def test_resolve_chat_completions_url_appends_path_to_base() -> None:default = "https://api.xiaomimimo.com/v1/chat/completions"assert _resolve_chat_completions_url(None, default) == default@@ -971,6 +970,60 @@ async def test_retries_on_every_advertised_transient_exception(assert post.await_count == 2[email protected]("provider_cls", [OpenAITranscriptionProvider, StepFunTranscriptionProvider])[email protected]("phase", ["request", "stream", "sleep"])+async def test_transcription_cancellation_releases_client(monkeypatch, audio_file, provider_cls, phase):+ entered, blocked = asyncio.Event(), asyncio.Event()+ requests, streams, waits = [], [], []++ class Stream(httpx.AsyncByteStream):+ closed = False++ async def __aiter__(self):+ if phase == "stream":+ entered.set()+ await blocked.wait()+ yield b""++ async def aclose(self):+ self.closed = True++ async def handle(request):+ requests.append(request)+ if phase == "request":+ entered.set()+ await blocked.wait()+ stream = Stream()+ streams.append(stream)+ return httpx.Response(200 if phase == "stream" else 503, stream=stream)++ client = httpx.AsyncClient(transport=httpx.MockTransport(handle))++ async def sleep(delay):+ waits.append(delay)+ assert streams[-1].closed+ entered.set()+ await blocked.wait()++ monkeypatch.setattr("nanobot.providers.transcription.httpx.AsyncClient", lambda: client)+ monkeypatch.setattr("nanobot.providers.transcription.asyncio.sleep", sleep)+ task = asyncio.create_task(provider_cls(api_key="sk-test").transcribe(audio_file))+ try:+ await asyncio.wait_for(entered.wait(), timeout=5)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ assert len(requests) == 1+ assert all(stream.closed for stream in streams)+ assert client.is_closed+ assert waits == ([1.0] if phase == "sleep" else [])+ finally:+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ await client.aclose()++# ---------------------------------------------------------------------------# apiBase normalization (#3637): a chat-style base must not be POSTed verbatim# ---------------------------------------------------------------------------diff --git a/tests/tools/test_web_search_tool.py b/tests/tools/test_web_search_tool.pyindex 4dc768b23..14fc1cbef 100644--- a/tests/tools/test_web_search_tool.py+++ b/tests/tools/test_web_search_tool.py@@ -1,5 +1,8 @@"""Tests for multi-provider web search."""+import asyncio+from contextlib import asynccontextmanager+import httpximport pytest@@ -97,7 +100,9 @@ async def test_brave_search_retries_rate_limit_once(monkeypatch):async def mock_get(self, url, **kw):calls["n"] += 1if calls["n"] == 1:- return _response(status=429, json={"error": "rate limit"})+ response = _response(status=429, json={"error": "rate limit"})+ response.headers["Retry-After"] = "3600"+ return responsereturn _response(json={"web": {"results": [{"title": "Recovered", "url": "https://example.com", "description": "ok"}]}})@@ -683,7 +688,9 @@ async def test_deepseek_search_retries_transient_errors_then_succeeds(async def mock_post(self, url, **kw):calls["n"] += 1if calls["n"] == 1:- return _response(status=status, json={"error": {"message": "transient"}})+ response = _response(status=status, json={"error": {"message": "transient"}})+ response.headers["Retry-After"] = "3600"+ return responsereturn _response(json={"results": [{"title": "Recovered", "url": "https://example.com/ok", "snippet": "ok"}]})@@ -1396,3 +1403,71 @@ async def test_olostep_package_missing_returns_install_hint(monkeypatch):assert result == ("Error: Olostep support is not installed. Run `nanobot plugins enable olostep`.")++[email protected]("provider,outcome", [+ ("brave", 500),+ ("brave", httpx.ReadTimeout("slow")),+ ("deepseek", httpx.ConnectError("offline")),+])+async def test_search_does_not_retry_unclassified_failures(monkeypatch, provider, outcome):+ calls = []++ async def request(*args, **kwargs):+ calls.append(args)+ if isinstance(outcome, Exception):+ raise outcome+ return _response(status=outcome)++ method = "get" if provider == "brave" else "post"+ monkeypatch.setattr(httpx.AsyncClient, method, request)+ result = await _tool(provider=provider, api_key="test-key").execute(query="failure")+ assert is_tool_error_result(result)+ assert len(calls) == 1++[email protected]("provider", ["brave", "deepseek"])[email protected]("phase", ["request", "backoff"])+async def test_search_task_cancellation_closes_client_without_replay(monkeypatch, provider, phase):+ entered, closed = asyncio.Event(), asyncio.Event()+ calls = []+ real_client = httpx.AsyncClient++ async def block():+ entered.set()+ await asyncio.Event().wait()++ async def handle(request):+ calls.append(request)+ if phase == "request":+ await block()+ return httpx.Response(429, json={})++ @asynccontextmanager+ async def client(**kwargs):+ instance = real_client(transport=httpx.MockTransport(handle), **kwargs)+ try:+ async with instance:+ yield instance+ finally:+ assert instance.is_closed+ closed.set()++ async def sleep(delay):+ assert delay == 1.0+ await block()++ monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", client)+ monkeypatch.setattr("nanobot.agent.tools.web.asyncio.sleep", sleep)+ task = asyncio.create_task(_tool(provider=provider, api_key="test-key").execute(query="cancel"))+ try:+ await asyncio.wait_for(entered.wait(), timeout=2)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ finally:+ if not task.done():+ task.cancel()+ await asyncio.gather(task, return_exceptions=True)+ assert len(calls) == 1+ assert closed.is_set()