Skip to content

Your lightweight open-source personal agent.

fix(retry): prevent replay amplification and isolate backoff (NAN-313)

chengyongru4e77d9c3f2bc
diff --git a/nanobot/agent/tools/mcp.py b/nanobot/agent/tools/mcp.py
index 87d92259e..197878b80 100644
--- a/nanobot/agent/tools/mcp.py
+++ b/nanobot/agent/tools/mcp.py
@@ -12,7 +12,7 @@ import traceback
import urllib.parse
from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping
from contextlib import AsyncExitStack, suppress
-from typing import TYPE_CHECKING, Any, Literal, Protocol, cast
+from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar, cast
import httpx
from loguru import logger
@@ -62,6 +62,7 @@ _HTTP_READ_TIMEOUT = 300.0
# Replace anything outside [a-zA-Z0-9_-] with underscore and collapse runs.
_SANITIZE_RE = re.compile(r"_+")
_ReconnectCallback = Callable[[str, str, Tool], Awaitable[Tool | None]]
+_MCPResultT = TypeVar("_MCPResultT")
MCPServerLoader = Callable[[], tuple[Mapping[str, "MCPServerConfig"], NetworkConfig]]
MCPRuntimeStatus = Literal["connecting", "connected", "failed"]
@@ -572,6 +573,75 @@ class _MCPWrapperBase(Tool):
self._session = refreshed_session
return True
+ async def _request_with_recovery(
+ self,
+ request: Callable[[], Awaitable[_MCPResultT]],
+ timeout: float,
+ capability_kind: Literal["tool", "resource", "prompt"],
+ ) -> _MCPResultT | ToolResult:
+ """Recover requests only; rendering and reconnect/backoff failures stay outside replay."""
+ from mcp.shared.exceptions import McpError
+
+ operation = f"{capability_kind} {'read' if capability_kind == 'resource' else 'call'}"
+ refreshed_session = False
+ retried = False
+ while True:
+ try:
+ return await asyncio.wait_for(request(), timeout=timeout)
+ except asyncio.TimeoutError:
+ logger.warning(
+ "MCP {} '{}' timed out after {}s", capability_kind, self._name, timeout
+ )
+ return ToolResult.error(f"(MCP {operation} timed out after {timeout}s)")
+ except asyncio.CancelledError:
+ # MCP SDK cancel scopes can leak cancellation without cancelling our task.
+ if task_is_cancelling():
+ raise
+ logger.warning("MCP {} '{}' was cancelled by server/SDK", capability_kind, self._name)
+ return ToolResult.error(f"(MCP {operation} was cancelled)")
+ except Exception as exc:
+ if await self._refresh_session_after_termination(
+ exc, refreshed_session, capability_kind
+ ):
+ refreshed_session = True
+ continue
+ if capability_kind == "prompt" and isinstance(exc, McpError):
+ logger.opt(exception=exc if tool_log_content_allowed() else False).error(
+ "MCP prompt '{}' failed: code={} message={}",
+ self._name,
+ exc.error.code,
+ exc.error.message if tool_log_content_allowed() else "[content hidden]",
+ )
+ return ToolResult.error(
+ f"(MCP prompt call failed: {exc.error.message} [code {exc.error.code}])"
+ )
+ if _is_transient(exc):
+ if not retried:
+ logger.warning(
+ "MCP {} '{}' hit transient error ({}), retrying once...",
+ capability_kind,
+ self._name,
+ type(exc).__name__,
+ )
+ retried = True
+ await asyncio.sleep(1)
+ continue
+ logger.opt(exception=exc if tool_log_content_allowed() else False).error(
+ "MCP {} '{}' failed after retry: {}",
+ capability_kind,
+ self._name,
+ type(exc).__name__,
+ )
+ return ToolResult.error(f"(MCP {operation} failed after retry: {type(exc).__name__})")
+ logger.opt(exception=exc if tool_log_content_allowed() else False).error(
+ "MCP {} '{}' failed: {}: {}",
+ capability_kind,
+ self._name,
+ type(exc).__name__,
+ exc if tool_log_content_allowed() else "[content hidden]",
+ )
+ return ToolResult.error(f"(MCP {operation} failed: {type(exc).__name__})")
+
def _image_block_data_url(block: Any, types: Any) -> str | None:
"""Return a base64 ``data:`` URL for an MCP image-bearing content block.
@@ -653,81 +723,29 @@ class MCPToolWrapper(_MCPWrapperBase):
return self._parameters
async def execute(self, **kwargs: Any) -> str:
- retried_transient = False
- refreshed_session = False
- while True:
- try:
- result = await asyncio.wait_for(
- self._session.call_tool(self._original_name, arguments=kwargs),
- timeout=self._tool_timeout,
- )
- except asyncio.TimeoutError:
- logger.warning(
- "MCP tool '{}' timed out after {}s", self._name, self._tool_timeout
- )
- return ToolResult.error(
- f"(MCP tool call timed out after {self._tool_timeout}s)"
- )
- except asyncio.CancelledError:
- # MCP SDK's anyio cancel scopes can leak CancelledError on timeout/failure.
- # Re-raise only if our task was externally cancelled (e.g. /stop).
- if task_is_cancelling():
- raise
- logger.warning("MCP tool '{}' was cancelled by server/SDK", self._name)
- return ToolResult.error("(MCP tool call was cancelled)")
- except Exception as exc:
- if await self._refresh_session_after_termination(
- exc,
- refreshed_session,
- "tool",
- ):
- refreshed_session = True
- continue
- if _is_transient(exc):
- if not retried_transient:
- retried_transient = True
- logger.warning(
- "MCP tool '{}' hit transient error ({}), retrying once...",
- self._name,
- type(exc).__name__,
- )
- await asyncio.sleep(1) # Brief backoff before retry
- continue
- # Second transient failure — give up with retry-specific message
- logger.opt(exception=tool_log_content_allowed()).error(
- "MCP tool '{}' failed after retry: {}",
- self._name,
- type(exc).__name__,
- )
- return ToolResult.error(
- f"(MCP tool call failed after retry: {type(exc).__name__})"
- )
- logger.opt(exception=tool_log_content_allowed()).error(
- "MCP tool '{}' failed: {}: {}",
- self._name,
- type(exc).__name__,
- exc if tool_log_content_allowed() else "[content hidden]",
- )
- return ToolResult.error(
- f"(MCP tool call failed: {type(exc).__name__})"
- )
- else:
- # Success — extract text and persist any image content as artifacts.
- try:
- rendered = self._render_call_result(result.content, kwargs)
- if getattr(result, "isError", False):
- return ToolResult.error(rendered)
- return rendered
- except Exception as exc:
- logger.opt(exception=tool_log_content_allowed()).error(
- "MCP tool '{}' failed while rendering result: {}: {}",
- self._name,
- type(exc).__name__,
- exc if tool_log_content_allowed() else "[content hidden]",
- )
- return ToolResult.error(
- f"(MCP tool returned malformed content: {type(exc).__name__})"
- )
+ result = await self._request_with_recovery(
+ lambda: self._session.call_tool(self._original_name, arguments=kwargs),
+ self._tool_timeout,
+ "tool",
+ )
+ if isinstance(result, ToolResult):
+ return result
+ # Success — extract text and persist any image content as artifacts.
+ try:
+ rendered = self._render_call_result(result.content, kwargs)
+ if getattr(result, "isError", False):
+ return ToolResult.error(rendered)
+ return rendered
+ except Exception as exc:
+ logger.opt(exception=tool_log_content_allowed()).error(
+ "MCP tool '{}' failed while rendering result: {}: {}",
+ self._name,
+ type(exc).__name__,
+ exc if tool_log_content_allowed() else "[content hidden]",
+ )
+ return ToolResult.error(
+ f"(MCP tool returned malformed content: {type(exc).__name__})"
+ )
def _render_call_result(self, content: Any, arguments: Mapping[str, Any]) -> str:
"""Turn MCP content blocks into a tool result string.
@@ -825,65 +843,22 @@ class MCPResourceWrapper(_MCPWrapperBase):
async def execute(self, **kwargs: Any) -> str:
from mcp import types
- retried_transient = False
- refreshed_session = False
- while True:
- try:
- result = await asyncio.wait_for(
- self._session.read_resource(self._uri),
- timeout=self._resource_timeout,
- )
- except asyncio.TimeoutError:
- logger.warning(
- "MCP resource '{}' timed out after {}s", self._name, self._resource_timeout
- )
- return f"(MCP resource read timed out after {self._resource_timeout}s)"
- except asyncio.CancelledError:
- if task_is_cancelling():
- raise
- logger.warning("MCP resource '{}' was cancelled by server/SDK", self._name)
- return "(MCP resource read was cancelled)"
- except Exception as exc:
- if await self._refresh_session_after_termination(
- exc,
- refreshed_session,
- "resource",
- ):
- refreshed_session = True
- continue
- if _is_transient(exc):
- if not retried_transient:
- retried_transient = True
- logger.warning(
- "MCP resource '{}' hit transient error ({}), retrying once...",
- self._name,
- type(exc).__name__,
- )
- await asyncio.sleep(1)
- continue
- logger.opt(exception=tool_log_content_allowed()).error(
- "MCP resource '{}' failed after retry: {}",
- self._name,
- type(exc).__name__,
- )
- return f"(MCP resource read failed after retry: {type(exc).__name__})"
- logger.opt(exception=tool_log_content_allowed()).error(
- "MCP resource '{}' failed: {}: {}",
- self._name,
- type(exc).__name__,
- exc if tool_log_content_allowed() else "[content hidden]",
- )
- return f"(MCP resource read failed: {type(exc).__name__})"
+ result = await self._request_with_recovery(
+ lambda: self._session.read_resource(self._uri),
+ self._resource_timeout,
+ "resource",
+ )
+ if isinstance(result, ToolResult):
+ return str(result)
+ parts: list[str] = []
+ for block in result.contents:
+ if isinstance(block, types.TextResourceContents):
+ parts.append(block.text)
+ elif isinstance(cast(object, block), types.BlobResourceContents):
+ parts.append(f"[Binary resource: {len(block.blob)} bytes]")
else:
- parts: list[str] = []
- for block in result.contents:
- if isinstance(block, types.TextResourceContents):
- parts.append(block.text)
- elif isinstance(cast(object, block), types.BlobResourceContents):
- parts.append(f"[Binary resource: {len(block.blob)} bytes]")
- else:
- parts.append(str(block))
- return "\n".join(parts) or "(no output)"
+ parts.append(str(block))
+ return "\n".join(parts) or "(no output)"
class MCPPromptWrapper(_MCPWrapperBase):
@@ -942,87 +917,28 @@ class MCPPromptWrapper(_MCPWrapperBase):
async def execute(self, **kwargs: Any) -> str:
from mcp import types
- from mcp.shared.exceptions import McpError
- retried_transient = False
- refreshed_session = False
- while True:
- try:
- result = await asyncio.wait_for(
- self._session.get_prompt(self._prompt_name, arguments=kwargs),
- timeout=self._prompt_timeout,
- )
- except asyncio.TimeoutError:
- logger.warning(
- "MCP prompt '{}' timed out after {}s", self._name, self._prompt_timeout
- )
- return f"(MCP prompt call timed out after {self._prompt_timeout}s)"
- except asyncio.CancelledError:
- if task_is_cancelling():
- raise
- logger.warning("MCP prompt '{}' was cancelled by server/SDK", self._name)
- return "(MCP prompt call was cancelled)"
- except McpError as exc:
- if await self._refresh_session_after_termination(
- exc,
- refreshed_session,
- "prompt",
- ):
- refreshed_session = True
- continue
- logger.opt(exception=tool_log_content_allowed()).error(
- "MCP prompt '{}' failed: code={} message={}",
- self._name,
- exc.error.code,
- exc.error.message if tool_log_content_allowed() else "[content hidden]",
- )
- return f"(MCP prompt call failed: {exc.error.message} [code {exc.error.code}])"
- except Exception as exc:
- if await self._refresh_session_after_termination(
- exc,
- refreshed_session,
- "prompt",
- ):
- refreshed_session = True
- continue
- if _is_transient(exc):
- if not retried_transient:
- retried_transient = True
- logger.warning(
- "MCP prompt '{}' hit transient error ({}), retrying once...",
- self._name,
- type(exc).__name__,
- )
- await asyncio.sleep(1)
- continue
- logger.opt(exception=tool_log_content_allowed()).error(
- "MCP prompt '{}' failed after retry: {}",
- self._name,
- type(exc).__name__,
- )
- return f"(MCP prompt call failed after retry: {type(exc).__name__})"
- logger.opt(exception=tool_log_content_allowed()).error(
- "MCP prompt '{}' failed: {}: {}",
- self._name,
- type(exc).__name__,
- exc if tool_log_content_allowed() else "[content hidden]",
- )
- return f"(MCP prompt call failed: {type(exc).__name__})"
- else:
- parts: list[str] = []
- for message in result.messages:
- content = message.content
- if isinstance(content, types.TextContent):
- parts.append(content.text)
- elif isinstance(content, list):
- for block in content:
- if isinstance(block, types.TextContent):
- parts.append(block.text)
- else:
- parts.append(str(block))
+ result = await self._request_with_recovery(
+ lambda: self._session.get_prompt(self._prompt_name, arguments=kwargs),
+ self._prompt_timeout,
+ "prompt",
+ )
+ if isinstance(result, ToolResult):
+ return str(result)
+ parts: list[str] = []
+ for message in result.messages:
+ content = message.content
+ if isinstance(content, types.TextContent):
+ parts.append(content.text)
+ elif isinstance(content, list):
+ for block in content:
+ if isinstance(block, types.TextContent):
+ parts.append(block.text)
else:
- parts.append(str(content))
- return "\n".join(parts) or "(no output)"
+ parts.append(str(block))
+ else:
+ parts.append(str(content))
+ return "\n".join(parts) or "(no output)"
async def connect_mcp_servers(
diff --git a/nanobot/channels/manager.py b/nanobot/channels/manager.py
index d6fe97d97..c7fc12476 100644
--- a/nanobot/channels/manager.py
+++ b/nanobot/channels/manager.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import hashlib
import inspect
+import random
import time
from collections import OrderedDict
from collections.abc import Awaitable, Callable, Iterable, Mapping
@@ -62,7 +63,7 @@ def _default_webui_dist() -> Path | None:
return candidate if candidate.is_dir() else None
-# Retry delays for message sending (exponential backoff: 1s, 2s, 4s)
+# Upper bounds for local equal-jitter send backoff (1s, 2s, then 4s).
_SEND_RETRY_DELAYS = (1, 2, 4)
_OUTBOUND_CONCURRENCY = 32
_OUTBOUND_PENDING_LIMIT = 256
@@ -797,8 +798,8 @@ class ChannelManager:
async def _queue_outbound(self, channel: BaseChannel, msg: OutboundMessage) -> None:
"""Bound scheduled sends; preserve FIFO per destination, not across chats.
- Retry waits occupy only their destination and one send slot. Saturation
- pauses dispatch rather than creating unbounded tasks or dropping messages.
+ Retry waits retain their destination tail and admission, not a send permit.
+ Saturation pauses dispatch rather than creating unbounded tasks or dropping messages.
"""
if msg.delivery_id in self._scheduled_deliveries:
return
@@ -816,11 +817,12 @@ class ChannelManager:
async def send() -> None:
if previous is not None:
await asyncio.shield(asyncio.gather(previous, return_exceptions=True))
- async with self._outbound_sends:
- if msg.delivery_id is not None:
+ if msg.delivery_id is not None:
+ # Acquire before claiming: queued cancellation must remain pending.
+ async with self._outbound_sends:
await self._send_scheduled(channel, msg)
- else:
- await self._send_with_retry(channel, msg)
+ else:
+ await self._send_with_retry(channel, msg)
task = asyncio.create_task(send(), name=f"outbound-{msg.channel}-{msg.chat_id}")
self._outbound_tasks[task] = key
@@ -1168,9 +1170,22 @@ class ChannelManager:
attempt = 0
while True:
- attempt += 1
try:
- await self._send_once(channel, msg)
+ async with asyncio.timeout_at(deadline):
+ await self._outbound_sends.acquire()
+ except TimeoutError:
+ logger.warning("Send to {} deadline expired waiting for a send slot", msg.channel)
+ return
+ try:
+ try:
+ if deadline is not None and asyncio.get_running_loop().time() >= deadline:
+ logger.warning("Send to {} deadline expired before submission", msg.channel)
+ return
+ attempt += 1
+ # The deadline bounds admission, not an already-submitted send.
+ await self._send_once(channel, msg)
+ finally:
+ self._outbound_sends.release()
return # Send succeeded
except asyncio.CancelledError:
raise # Propagate cancellation for graceful shutdown
@@ -1195,7 +1210,8 @@ class ChannelManager:
msg.channel, attempt,
)
return
- delay = _SEND_RETRY_DELAYS[min(attempt - 1, len(_SEND_RETRY_DELAYS) - 1)]
+ base = _SEND_RETRY_DELAYS[min(attempt - 1, len(_SEND_RETRY_DELAYS) - 1)]
+ delay = random.uniform(base / 2, base)
if deadline is not None:
delay = min(delay, max(0.0, deadline - loop.time()))
attempt_label = str(attempt)
diff --git a/nanobot/channels/telegram/runtime.py b/nanobot/channels/telegram/runtime.py
index fb9fa00ec..2220e54e5 100644
--- a/nanobot/channels/telegram/runtime.py
+++ b/nanobot/channels/telegram/runtime.py
@@ -1136,6 +1136,11 @@ class TelegramChannel(BaseChannel):
buf.message_id = sent.message_id
buf.draft_id = None
+ def should_retry_send_error(self, error: Exception) -> bool:
+ # Per-part API retries belong to this adapter. Whole-message replay is safe
+ # only before submission; otherwise it may duplicate a successful prefix.
+ return isinstance(error, ChannelNotReadyError)
+
async def send(self, msg: OutboundMessage) -> None:
"""Send a message through Telegram."""
if isinstance(msg.event, ContextCompactionEvent) and not (
diff --git a/nanobot/channels/weixin/tests/test_weixin_hardening.py b/nanobot/channels/weixin/tests/test_weixin_hardening.py
index d30e3e80a..917387f83 100644
--- a/nanobot/channels/weixin/tests/test_weixin_hardening.py
+++ b/nanobot/channels/weixin/tests/test_weixin_hardening.py
@@ -118,6 +118,7 @@ async def test_channel_manager_does_not_retry_permanent_weixin_error(monkeypatch
manager = ChannelManager.__new__(ChannelManager)
manager.config = Config.model_validate({"channels": {"sendMaxRetries": 3}})
manager.bus = MessageBus()
+ manager._outbound_sends = asyncio.Semaphore(32)
channel = _channel()
channel.send = AsyncMock(
side_effect=WeixinAPIError(
diff --git a/nanobot/channels/whatsapp_agent/tests/test_runtime.py b/nanobot/channels/whatsapp_agent/tests/test_runtime.py
index 00aece1cb..5b81af546 100644
--- a/nanobot/channels/whatsapp_agent/tests/test_runtime.py
+++ b/nanobot/channels/whatsapp_agent/tests/test_runtime.py
@@ -667,6 +667,7 @@ def test_manager_retry_resumes_after_acknowledged_parts(
monkeypatch.setattr("nanobot.channels.manager._SEND_RETRY_DELAYS", (0,))
manager = object.__new__(ChannelManager)
manager.config = Config()
+ manager._outbound_sends = asyncio.Semaphore(32)
message = OutboundMessage(channel=channel.name, chat_id="creator", content="A" * 4096 + "B")
asyncio.run(manager._send_with_retry(channel, message))
assert delivered == ["A" * 4096, "B"]
@@ -764,6 +765,7 @@ def test_media_failure_never_replays_accepted_text_or_attachments(
monkeypatch.setattr("nanobot.channels.manager._SEND_RETRY_DELAYS", (0,))
manager = object.__new__(ChannelManager)
manager.config = Config()
+ manager._outbound_sends = asyncio.Semaphore(32)
message = OutboundMessage(
channel=channel.name, chat_id="creator", content="Files",
media=[str(attachment), str(attachment)],
diff --git a/nanobot/providers/base.py b/nanobot/providers/base.py
index 429dadbd3..d5a4c5040 100644
--- a/nanobot/providers/base.py
+++ b/nanobot/providers/base.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import json
import os
+import random
import re
import time
from abc import ABC, abstractmethod
@@ -1590,34 +1591,29 @@ class LLMProvider(ABC):
should_retry_guard: Callable[[], bool] | None = None,
on_stream_recover: Callable[[], Awaitable[None]] | None = None,
) -> LLMResponse:
- attempt = 0
- delays = list(self._CHAT_RETRY_DELAYS)
+ response = await call(**kw)
+ if response.finish_reason != "error":
+ return response
+
+ attempt = 1
+ delays = self._CHAT_RETRY_DELAYS
persistent = retry_mode == "persistent"
- last_response: LLMResponse | None = None
+ max_attempts = None if persistent else len(delays) + 1
last_error_key: str | None = None
identical_error_count = 0
-
async def _finish_retry_status(
- state: Literal["recovered", "cleared"],
- response: LLMResponse,
+ state: Literal["recovered", "cleared", "exhausted"],
+ result: LLMResponse,
) -> None:
- if attempt > 1 and on_retry_status:
- await on_retry_status(
- RetryStatusEvent(
- state=state,
- attempt=attempt,
- max_attempts=None if persistent else len(delays) + 1,
- error_kind=self.public_error_kind(response),
- )
- )
+ if on_retry_status and (attempt > 1 or state == "exhausted"):
+ await on_retry_status(RetryStatusEvent(
+ state=state,
+ attempt=attempt,
+ max_attempts=max_attempts,
+ error_kind=self.public_error_kind(result),
+ ))
while True:
- attempt += 1
- response = await call(**kw)
- if response.finish_reason != "error":
- await _finish_retry_status("recovered", response)
- return response
- last_response = response
if should_retry_guard is not None and not should_retry_guard():
is_timeout = (response.error_kind or "").lower() == "timeout"
if is_timeout:
@@ -1632,7 +1628,6 @@ class LLMProvider(ABC):
"LLM stream stalled after content was emitted; "
"suppressing delta callbacks and retrying"
)
- kw.setdefault("on_content_delta", None)
kw["on_content_delta"] = None
kw["on_thinking_delta"] = None
kw["on_tool_call_delta"] = None
@@ -1651,106 +1646,29 @@ class LLMProvider(ABC):
identical_error_count = 1 if error_key else 0
if not self.is_transient_response(response):
- # Removing the user's images cannot repair account access.
- # Do not rebuild native provider state for an unrelated failure.
- if (
- (response.error_kind or "").lower() in {"authentication", "oauth_auth_required"}
- or response.error_status_code in {401, 403}
- or self.is_arrearage_response(response)
- ):
- await _finish_retry_status("cleared", response)
- return response
- stripped = self._strip_image_content(kw["messages"])
- provider_context = kw.get("provider_context")
- stripped_context: ProviderCallContext | None = None
- if isinstance(provider_context, ProviderCallContext):
- state = provider_context.conversation_state
- if state is not None and (
- stripped is not None
- or self._strip_image_content(state.pending_messages) is not None
- or self._contains_image_content(state.payload)
- ):
- # Provider-owned payloads may retain earlier input_image items.
- # Rebuild from the stripped public transcript for this retry.
- stripped_context = ProviderCallContext(
- context_window_tokens=(
- provider_context.context_window_tokens
- ),
- session_id=provider_context.session_id,
- events=provider_context.events,
- response_preset=provider_context.response_preset,
- response_is_fallback=provider_context.response_is_fallback,
- )
- if stripped is not None or stripped_context is not None:
- logger.warning(
- "Non-transient LLM error with image content, retrying without images"
- )
- retry_kw = dict(kw)
- if stripped is not None:
- retry_kw["messages"] = stripped
- if stripped_context is not None:
- retry_kw["provider_context"] = stripped_context
- result = await call(**retry_kw)
- # Permanently strip images from the original messages so
- # subsequent iterations do not repeat the error-retry cycle.
- if result.finish_reason != "error":
- self._strip_image_content_inplace(original_messages)
- await _finish_retry_status(
- "recovered" if result.finish_reason != "error" else "cleared",
- result,
- )
- return result
- await _finish_retry_status("cleared", response)
- return response
-
- if persistent and identical_error_count >= self._PERSISTENT_IDENTICAL_ERROR_LIMIT:
- logger.warning(
- "Stopping persistent retry after {} identical transient errors: {}",
- identical_error_count,
- (response.content or "")[:120].lower(),
+ response = await self._retry_without_images(
+ call, kw, original_messages, response,
+ )
+ await _finish_retry_status(
+ "recovered" if response.finish_reason != "error" else "cleared",
+ response,
)
- if on_retry_exhausted:
- await on_retry_exhausted(
- f"Persistent retry stopped after {identical_error_count} identical errors."
- )
- if on_retry_status:
- await on_retry_status(
- RetryStatusEvent(
- state="exhausted",
- attempt=attempt,
- max_attempts=None,
- error_kind=self.public_error_kind(response),
- )
- )
return response
- if not persistent and attempt > len(delays):
- logger.warning(
- "LLM request failed after {} attempts, giving up: {}",
- attempt,
- (response.content or "")[:120].lower(),
- )
- if on_retry_exhausted:
- await on_retry_exhausted(
- f"Model request failed after {attempt} attempts, giving up."
- )
- if on_retry_status:
- await on_retry_status(
- RetryStatusEvent(
- state="exhausted",
- attempt=attempt,
- max_attempts=len(delays) + 1,
- error_kind=self.public_error_kind(response),
- )
- )
+ if (
+ persistent and identical_error_count >= self._PERSISTENT_IDENTICAL_ERROR_LIMIT
+ or not persistent and attempt > len(delays)
+ ):
break
retry_after = self._extract_retry_after_from_response(response)
base_delay = delays[min(attempt - 1, len(delays) - 1)]
- delay = retry_after + RETRY_AFTER_BUFFER if retry_after else base_delay
+ delay = (
+ retry_after + RETRY_AFTER_BUFFER
+ if retry_after else random.uniform(base_delay / 2, base_delay)
+ )
if persistent:
delay = min(delay, self._PERSISTENT_MAX_DELAY)
-
logger.warning(
"LLM transient error (attempt {}{}), retrying in {}s: {}",
attempt,
@@ -1763,12 +1681,83 @@ class LLMProvider(ABC):
attempt=attempt,
persistent=persistent,
error_kind=self.public_error_kind(response),
- max_attempts=None if persistent else len(delays) + 1,
+ max_attempts=max_attempts,
on_retry_wait=on_retry_wait,
on_retry_status=on_retry_status,
)
+ attempt += 1
+ response = await call(**kw)
+ if response.finish_reason != "error":
+ await _finish_retry_status("recovered", response)
+ return response
+
+ if persistent:
+ logger.warning(
+ "Stopping persistent retry after {} identical transient errors: {}",
+ identical_error_count,
+ (response.content or "")[:120].lower(),
+ )
+ exhausted_message = (
+ f"Persistent retry stopped after {identical_error_count} identical errors."
+ )
+ else:
+ logger.warning(
+ "LLM request failed after {} attempts, giving up: {}",
+ attempt,
+ (response.content or "")[:120].lower(),
+ )
+ exhausted_message = f"Model request failed after {attempt} attempts, giving up."
+ if on_retry_exhausted:
+ await on_retry_exhausted(exhausted_message)
+ await _finish_retry_status("exhausted", response)
+ return response
- return last_response if last_response is not None else await call(**kw) # pyright: ignore[reportUnnecessaryComparison]
+ async def _retry_without_images(
+ self,
+ call: Callable[..., Awaitable[LLMResponse]],
+ kw: dict[str, Any],
+ original_messages: list[dict[str, Any]],
+ response: LLMResponse,
+ ) -> LLMResponse:
+ """Try image compatibility repair once, outside transient retries."""
+ # Removing the user's images cannot repair account access.
+ if (
+ (response.error_kind or "").lower() in {"authentication", "oauth_auth_required"}
+ or response.error_status_code in {401, 403}
+ or self.is_arrearage_response(response)
+ ):
+ return response
+ stripped = self._strip_image_content(kw["messages"])
+ provider_context = kw.get("provider_context")
+ stripped_context: ProviderCallContext | None = None
+ if isinstance(provider_context, ProviderCallContext):
+ state = provider_context.conversation_state
+ if state is not None and (
+ stripped is not None
+ or self._strip_image_content(state.pending_messages) is not None
+ or self._contains_image_content(state.payload)
+ ):
+ # Native payloads may retain images: rebuild from the public transcript.
+ stripped_context = ProviderCallContext(
+ context_window_tokens=provider_context.context_window_tokens,
+ session_id=provider_context.session_id,
+ events=provider_context.events,
+ response_preset=provider_context.response_preset,
+ response_is_fallback=provider_context.response_is_fallback,
+ )
+ if stripped is None and stripped_context is None:
+ return response
+ logger.warning("Non-transient LLM error with image content, retrying without images")
+ retry_kw = dict(kw)
+ if stripped is not None:
+ retry_kw["messages"] = stripped
+ if stripped_context is not None:
+ retry_kw["provider_context"] = stripped_context
+ result = await call(**retry_kw)
+ # Keep later iterations from repeating the same image repair.
+ if result.finish_reason != "error":
+ self._strip_image_content_inplace(original_messages)
+ return result
async def aclose(self) -> None:
"""Close provider-owned transports after their active requests have stopped."""
diff --git a/nanobot/providers/bedrock_provider.py b/nanobot/providers/bedrock_provider.py
index d7ba30e5d..3b50dd53c 100644
--- a/nanobot/providers/bedrock_provider.py
+++ b/nanobot/providers/bedrock_provider.py
@@ -90,7 +90,10 @@ class BedrockProvider(LLMProvider):
idle_timeout_s = resolve_stream_idle_timeout_s()
client_kwargs: dict[str, Any] = {
- "config": Config(connect_timeout=idle_timeout_s, read_timeout=idle_timeout_s),
+ "config": Config(
+ connect_timeout=idle_timeout_s, read_timeout=idle_timeout_s,
+ retries={"total_max_attempts": 1},
+ ),
}
if self.region:
client_kwargs["region_name"] = self.region
diff --git a/nanobot/providers/transcription.py b/nanobot/providers/transcription.py
index eee72dacf..8a1f26842 100644
--- a/nanobot/providers/transcription.py
+++ b/nanobot/providers/transcription.py
@@ -350,77 +350,71 @@ async def _post_stepfun_asr_with_retry(
}
async with httpx.AsyncClient() as client:
- for attempt in range(_MAX_RETRIES + 1):
- try:
- async with client.stream(
- "POST", url, headers=headers, json=body, timeout=60.0
- ) as resp:
- if resp.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES:
- logger.warning(
- "{} transcription transient HTTP {} (attempt {}/{})",
- provider_label,
- resp.status_code,
- attempt + 1,
- _MAX_RETRIES + 1,
- )
- await asyncio.sleep(_BACKOFF_S[attempt])
+ async def request_once() -> str | None:
+ # The entire stream closes before the owner decides whether to wait.
+ async with client.stream(
+ "POST", url, headers=headers, json=body, timeout=60.0
+ ) as resp:
+ resp.raise_for_status()
+ async for line in resp.aiter_lines():
+ if not line.startswith("data:"):
+ continue
+ payload_str = line[len("data:") :].strip()
+ if not payload_str:
continue
- resp.raise_for_status()
- final_text = None
- async for line in resp.aiter_lines():
- if not line.startswith("data:"):
- continue
- payload_str = line[len("data:") :].strip()
- if not payload_str:
- continue
- try:
- payload = json.loads(payload_str)
- except (json.JSONDecodeError, ValueError):
- continue
- payload = cast(dict[str, Any], payload)
- event_type = payload.get("type", "")
- if event_type == "error":
- msg = payload.get("message", "unknown error")
- logger.error("{} ASR error: {}", provider_label, msg)
- return ""
- if event_type == "transcript.text.done":
- final_text = payload.get("text", "")
- break
- if final_text is not None:
- return final_text
- # Stream ended without a final event — retry if attempts remain
- if attempt < _MAX_RETRIES:
- logger.warning(
- "{} transcription: no final event (attempt {}/{})",
- provider_label,
- attempt + 1,
- _MAX_RETRIES + 1,
- )
- await asyncio.sleep(_BACKOFF_S[attempt])
+ try:
+ payload = json.loads(payload_str)
+ except (json.JSONDecodeError, ValueError):
continue
+ payload = cast(dict[str, Any], payload)
+ event_type = payload.get("type", "")
+ if event_type == "error":
+ msg = payload.get("message", "unknown error")
+ logger.error("{} ASR error: {}", provider_label, msg)
+ return ""
+ if event_type == "transcript.text.done":
+ return payload.get("text", "")
+ return None
+
+ for attempt in range(_MAX_RETRIES + 1):
+ try:
+ final_text = await request_once()
+ if final_text is not None:
+ return final_text
+ if attempt == _MAX_RETRIES:
logger.error(
"{} transcription: stream ended without final text after {} attempts",
provider_label,
_MAX_RETRIES + 1,
)
return ""
+ logger.warning(
+ "{} transcription: no final event (attempt {}/{})",
+ provider_label,
+ attempt + 1,
+ _MAX_RETRIES + 1,
+ )
except httpx.HTTPStatusError as e:
- if e.response.status_code in _RETRYABLE_STATUS and attempt < _MAX_RETRIES:
- await asyncio.sleep(_BACKOFF_S[attempt])
- continue
- logger.error(
- "{} transcription HTTP {}{}",
+ if e.response.status_code not in _RETRYABLE_STATUS or attempt == _MAX_RETRIES:
+ logger.error(
+ "{} transcription HTTP {}{}",
+ provider_label,
+ e.response.status_code,
+ f" {e.response.reason_phrase}" if e.response.reason_phrase else "",
+ )
+ return ""
+ logger.warning(
+ "{} transcription transient HTTP {} (attempt {}/{})",
provider_label,
e.response.status_code,
- f" {e.response.reason_phrase}" if e.response.reason_phrase else "",
+ attempt + 1,
+ _MAX_RETRIES + 1,
)
- return ""
- except (httpx.RequestError, Exception):
- if attempt < _MAX_RETRIES:
- await asyncio.sleep(_BACKOFF_S[attempt])
- continue
- logger.exception("{} transcription request error", provider_label)
- return ""
+ except Exception:
+ if attempt == _MAX_RETRIES:
+ logger.exception("{} transcription request error", provider_label)
+ return ""
+ await asyncio.sleep(_BACKOFF_S[attempt])
return ""
diff --git a/tests/agent/test_mcp_transient_retry.py b/tests/agent/test_mcp_transient_retry.py
index bd9b4e723..ca3f423dc 100644
--- a/tests/agent/test_mcp_transient_retry.py
+++ b/tests/agent/test_mcp_transient_retry.py
@@ -5,10 +5,12 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
+from loguru import logger
from mcp import types as mcp_types
from mcp.shared.exceptions import McpError
from mcp.types import ErrorData
+from nanobot.agent.tools.context import RequestContext, request_context
from nanobot.agent.tools.mcp import (
MCPPromptWrapper,
MCPResourceWrapper,
@@ -491,3 +493,135 @@ async def test_prompt_reconnects_on_connection_closed_exception():
assert output == "fresh prompt"
assert old_session.get_prompt.call_count == 1
assert new_session.get_prompt.call_count == 1
+
+
+async def test_tool_refresh_leaves_one_transient_retry_available():
+ old, fresh = AsyncMock(), AsyncMock()
+ old.call_tool.side_effect = ConnectionResetError("old")
+ fresh.call_tool.side_effect = [ConnectionResetError("fresh"), _make_tool_result("ok")]
+ wrapper = MCPToolWrapper(old, "srv", _make_tool_def())
+ reconnect = AsyncMock(return_value=MCPToolWrapper(fresh, "srv", _make_tool_def()))
+ wrapper.set_reconnect_handler(reconnect)
+
+ with patch("nanobot.agent.tools.mcp.asyncio.sleep", new_callable=AsyncMock) as sleep:
+ assert await wrapper.execute(value="unchanged") == "ok"
+
+ old.call_tool.assert_awaited_once_with("test_tool", arguments={"value": "unchanged"})
+ assert fresh.call_tool.await_count == 2
+ reconnect.assert_awaited_once_with("srv", wrapper.name, wrapper)
+ sleep.assert_awaited_once_with(1)
+
+
+async def test_tool_later_refresh_does_not_reset_transient_budget():
+ old, fresh = AsyncMock(), AsyncMock()
+ old.call_tool.side_effect = [ConnectionResetError("first"), _session_terminated_error()]
+ fresh.call_tool.side_effect = ConnectionResetError("last")
+ wrapper = MCPToolWrapper(old, "srv", _make_tool_def())
+ replacement = MCPToolWrapper(fresh, "srv", _make_tool_def())
+ reconnect = AsyncMock(side_effect=[None, replacement])
+ wrapper.set_reconnect_handler(reconnect)
+
+ with patch("nanobot.agent.tools.mcp.asyncio.sleep", new_callable=AsyncMock) as sleep:
+ output = await wrapper.execute()
+
+ assert output == "(MCP tool call failed after retry: ConnectionResetError)"
+ assert is_tool_error_result(output)
+ assert (old.call_tool.await_count, fresh.call_tool.await_count) == (2, 1)
+ assert reconnect.await_count == 2
+ sleep.assert_awaited_once_with(1)
+
+
[email protected]("phase", ["refresh", "backoff"])
+async def test_tool_real_cancellation_during_recovery_propagates(phase):
+ entered, finished = asyncio.Event(), asyncio.Event()
+
+ async def block(*_args):
+ entered.set()
+ try:
+ await asyncio.Event().wait()
+ finally:
+ finished.set()
+
+ session = AsyncMock()
+ session.call_tool.side_effect = ConnectionResetError("gone")
+ wrapper = MCPToolWrapper(session, "srv", _make_tool_def())
+ if phase == "refresh":
+ wrapper.set_reconnect_handler(block)
+ with patch("nanobot.agent.tools.mcp.asyncio.sleep", side_effect=block):
+ task = asyncio.create_task(wrapper.execute())
+ try:
+ await asyncio.wait_for(entered.wait(), timeout=2)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ finally:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ assert task.cancelled() and finished.is_set()
+ session.call_tool.assert_awaited_once()
+
+
+async def test_tool_reconnect_failure_is_not_a_retryable_request_failure():
+ session = AsyncMock()
+ session.call_tool.side_effect = ConnectionResetError("request")
+ wrapper = MCPToolWrapper(session, "srv", _make_tool_def())
+ failure = ConnectionResetError("refresh")
+ reconnect = AsyncMock(side_effect=failure)
+ wrapper.set_reconnect_handler(reconnect)
+ with patch("nanobot.agent.tools.mcp.asyncio.sleep", new_callable=AsyncMock) as sleep:
+ with pytest.raises(ConnectionResetError) as raised:
+ await wrapper.execute()
+ assert raised.value is failure
+ session.call_tool.assert_awaited_once()
+ reconnect.assert_awaited_once()
+ sleep.assert_not_awaited()
+
+
+async def test_tool_transient_render_failure_cannot_replay_a_successful_request():
+ session = AsyncMock()
+ session.call_tool.return_value = _make_tool_result("ok")
+ wrapper = MCPToolWrapper(session, "srv", _make_tool_def())
+ reconnect = AsyncMock()
+ wrapper.set_reconnect_handler(reconnect)
+ with (
+ patch.object(wrapper, "_render_call_result", side_effect=ConnectionResetError("Session terminated")),
+ patch("nanobot.agent.tools.mcp.asyncio.sleep", new_callable=AsyncMock) as sleep,
+ ):
+ output = await wrapper.execute()
+ assert output == "(MCP tool returned malformed content: ConnectionResetError)"
+ assert is_tool_error_result(output)
+ session.call_tool.assert_awaited_once()
+ reconnect.assert_not_awaited()
+ sleep.assert_not_awaited()
+
+
[email protected]("kind", ["tool", "prompt"])
+async def test_request_error_logs_hide_content_and_prompt_retains_error_detail(kind):
+ session = AsyncMock()
+ if kind == "prompt":
+ session.get_prompt.side_effect = McpError(ErrorData(code=-32602, message="private-request-content"))
+ wrapper = MCPPromptWrapper(session, "srv", _make_prompt_def())
+ old = AsyncMock()
+ old.get_prompt.side_effect = _session_terminated_error()
+ stale = MCPPromptWrapper(old, "srv", _make_prompt_def())
+ stale.set_reconnect_handler(AsyncMock(return_value=wrapper))
+ wrapper = stale
+ expected = "(MCP prompt call failed: private-request-content [code -32602])"
+ else:
+ session.call_tool.side_effect = ValueError("private-request-content")
+ wrapper = MCPToolWrapper(session, "srv", _make_tool_def())
+ expected = "(MCP tool call failed: ValueError)"
+ records = []
+ sink = logger.add(lambda message: records.append(message.record), level="DEBUG")
+ try:
+ with request_context(RequestContext(channel="test", chat_id="test", log_content=False)):
+ output = await wrapper.execute()
+ finally:
+ logger.remove(sink)
+ assert output == expected
+ assert is_tool_error_result(output) is (kind == "tool")
+ error = next(record for record in records if record["level"].name == "ERROR")
+ assert "[content hidden]" in error["message"]
+ assert error["exception"] is None
+ assert all("private-request-content" not in record["message"] for record in records)
diff --git a/tests/agent/test_runner_fallback.py b/tests/agent/test_runner_fallback.py
index 6eff3a41e..f162f948a 100644
--- a/tests/agent/test_runner_fallback.py
+++ b/tests/agent/test_runner_fallback.py
@@ -4,7 +4,7 @@ from __future__ import annotations
import asyncio
from typing import Any
-from unittest.mock import AsyncMock, MagicMock, call, patch
+from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
@@ -476,7 +476,11 @@ class TestFallbackWhenPrimaryRaises:
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
factory.assert_called_once_with(_fallback("fallback-a"))
- assert sleep.await_args_list == [call(1), call(2), call(4)]
+ assert sleep.await_count == 3
+ assert all(
+ ceiling / 2 <= request.args[0] <= ceiling
+ for request, ceiling in zip(sleep.await_args_list, (1, 2, 4))
+ )
@pytest.mark.asyncio
async def test_authentication_exception_message_is_classified(self) -> None:
@@ -927,7 +931,11 @@ class TestFallbackOnPrimaryError:
assert result.content == "fallback ok"
assert fallback_models == ["fallback-b"]
- assert sleep.await_args_list == [call(1), call(2), call(4)]
+ assert sleep.await_count == 3
+ assert all(
+ ceiling / 2 <= request.args[0] <= ceiling
+ for request, ceiling in zip(sleep.await_args_list, (1, 2, 4))
+ )
@pytest.mark.asyncio
@pytest.mark.parametrize(("provider_name", "status", "code", "kind", "expected"), [
diff --git a/tests/channels/test_channel_manager_concurrency.py b/tests/channels/test_channel_manager_concurrency.py
index b053a890c..99ef97a81 100644
--- a/tests/channels/test_channel_manager_concurrency.py
+++ b/tests/channels/test_channel_manager_concurrency.py
@@ -244,3 +244,132 @@ async def test_dispatcher_shutdown_cleans_active_and_waiting_destination_tasks()
channel.send.assert_awaited_once()
assert not manager._outbound_tasks
assert not manager._outbound_tails
+
+
+async def test_all_send_slots_in_backoff_leave_healthy_chat_and_fifo_live(monkeypatch):
+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (0.6,))
+ manager, channel = make_manager()
+ failed = set()
+ delivered = []
+
+ async def send(msg):
+ if msg.chat_id != "healthy" and msg.chat_id not in failed:
+ failed.add(msg.chat_id)
+ raise OSError("temporary failure")
+ delivered.append((msg.chat_id, msg.content))
+
+ channel.send.side_effect = send
+ channel.should_retry_send_error.return_value = True
+ try:
+ for i in range(manager_module._OUTBOUND_CONCURRENCY):
+ await manager._queue_outbound(channel, OutboundMessage("websocket", str(i), "first"))
+ await wait_until(lambda: len(failed) == manager_module._OUTBOUND_CONCURRENCY)
+ await manager._queue_outbound(channel, OutboundMessage("websocket", "0", "last"))
+ await manager._queue_outbound(channel, OutboundMessage("websocket", "healthy", "ok"))
+ async with asyncio.timeout(0.2):
+ while ("healthy", "ok") not in delivered:
+ await asyncio.sleep(0)
+ assert ("0", "last") not in delivered
+ await wait_until(lambda: not manager._outbound_tasks)
+ assert [content for chat, content in delivered if chat == "0"] == ["first", "last"]
+ finally:
+ await manager._cancel_outbound()
+ assert not manager._outbound_tails
+
+
+async def test_scheduled_cancel_waiting_for_permit_does_not_claim(tmp_path, monkeypatch):
+ from nanobot.bus.delivery import DeliveryRequest
+
+ monkeypatch.setattr(manager_module, "_OUTBOUND_CONCURRENCY", 1)
+ config = Config.model_validate({"channels": {"websocket": {"enabled": False}}})
+ manager = ChannelManager(config, MessageBus(), config_path=tmp_path / "config.json")
+ channel = MagicMock(spec=BaseChannel, send=AsyncMock())
+ manager.channels["websocket"] = channel
+ msg = OutboundMessage("websocket", "scheduled", "result",
+ delivery=DeliveryRequest("run", "websocket:scheduled", None))
+ manager._outbox.record(msg)
+ claim = MagicMock(wraps=manager._outbox.claim)
+ monkeypatch.setattr(manager._outbox, "claim", claim)
+ await manager._outbound_sends.acquire()
+ try:
+ await manager._queue_outbound(channel, msg)
+ await asyncio.sleep(0)
+ await manager._cancel_outbound()
+ claim.assert_not_called()
+ channel.send.assert_not_awaited()
+ assert [item.delivery_id for item in manager._outbox.ready()] == [msg.delivery_id]
+ assert not manager._scheduled_deliveries
+ finally:
+ manager._outbound_sends.release()
+ await manager._cancel_outbound()
+
+
+async def test_restart_notice_does_not_outwait_send_permit_deadline(tmp_path):
+ from nanobot.utils.restart import RestartNotice
+
+ config = Config.model_validate({"channels": {"websocket": {"enabled": False}}})
+ manager = ChannelManager(config, MessageBus(), config_path=tmp_path / "config.json")
+ channel = MagicMock(spec=BaseChannel, is_running=True)
+ active = 0
+ release = asyncio.Event()
+ notices = []
+
+ async def send(msg):
+ nonlocal active
+ if msg.chat_id == "notice":
+ notices.append(msg)
+ return
+ active += 1
+ try:
+ await release.wait()
+ finally:
+ active -= 1
+
+ channel.send = AsyncMock(side_effect=send)
+ manager.channels["probe"] = channel
+ try:
+ for i in range(manager_module._OUTBOUND_CONCURRENCY):
+ await manager._queue_outbound(channel, OutboundMessage("probe", str(i), "blocked"))
+ await wait_until(lambda: active == manager_module._OUTBOUND_CONCURRENCY)
+ await asyncio.wait_for(manager._send_restart_notice_when_started(
+ RestartNotice(channel="probe", chat_id="notice", started_at_raw=""), timeout_s=0.05,
+ ), 1)
+ assert notices == []
+ assert active == manager_module._OUTBOUND_CONCURRENCY
+ finally:
+ await manager._cancel_outbound()
+ assert active == 0
+ assert manager._outbound_sends._value == manager_module._OUTBOUND_CONCURRENCY
+
+
+async def test_restart_notice_deadline_does_not_cancel_submitted_send(tmp_path):
+ from nanobot.utils.restart import RestartNotice
+
+ config = Config.model_validate({"channels": {"websocket": {"enabled": False}}})
+ manager = ChannelManager(config, MessageBus(), config_path=tmp_path / "config.json")
+ started = asyncio.Event()
+ release = asyncio.Event()
+
+ async def send(msg):
+ started.set()
+ await release.wait()
+
+ channel = MagicMock(spec=BaseChannel, is_running=True, send=AsyncMock(side_effect=send))
+ manager.channels["probe"] = channel
+ task = asyncio.create_task(manager._send_restart_notice_when_started(
+ RestartNotice(channel="probe", chat_id="notice", started_at_raw=""), timeout_s=0.05,
+ ))
+ try:
+ await asyncio.wait_for(started.wait(), 1)
+ try:
+ await asyncio.wait_for(asyncio.shield(task), 0.1)
+ except TimeoutError:
+ pass
+ assert not task.done()
+ release.set()
+ await asyncio.wait_for(task, 1)
+ channel.send.assert_awaited_once()
+ assert manager._outbound_sends._value == manager_module._OUTBOUND_CONCURRENCY
+ finally:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
diff --git a/tests/channels/test_channel_plugins.py b/tests/channels/test_channel_plugins.py
index dfe327faa..533541ad5 100644
--- a/tests/channels/test_channel_plugins.py
+++ b/tests/channels/test_channel_plugins.py
@@ -2997,6 +2997,7 @@ async def test_send_with_retry_succeeds_first_try():
mgr.bus = MessageBus()
mgr.channels = {"failing": _FailingChannel(fake_config, mgr.bus)}
mgr._dispatch_task = None
+ mgr._outbound_sends = asyncio.Semaphore(32)
msg = OutboundMessage(channel="failing", chat_id="123", content="test")
await mgr._send_with_retry(mgr.channels["failing"], msg)
@@ -3034,6 +3035,7 @@ async def test_send_with_retry_retries_on_failure():
mgr.bus = MessageBus()
mgr.channels = {"failing": _FailingChannel(fake_config, mgr.bus)}
mgr._dispatch_task = None
+ mgr._outbound_sends = asyncio.Semaphore(32)
msg = OutboundMessage(channel="failing", chat_id="123", content="test")
@@ -3075,6 +3077,7 @@ async def test_send_with_retry_no_retry_when_max_is_zero():
mgr.bus = MessageBus()
mgr.channels = {"failing": _FailingChannel(fake_config, mgr.bus)}
mgr._dispatch_task = None
+ mgr._outbound_sends = asyncio.Semaphore(32)
msg = OutboundMessage(channel="failing", chat_id="123", content="test")
@@ -3125,6 +3128,7 @@ async def test_send_with_retry_calls_send_delta():
mgr.bus = MessageBus()
mgr.channels = {"streaming": _StreamingChannel(fake_config, mgr.bus)}
mgr._dispatch_task = None
+ mgr._outbound_sends = asyncio.Semaphore(32)
msg = outbound_message_for_event(
channel="streaming",
@@ -3193,6 +3197,7 @@ async def test_send_with_retry_skips_send_when_streamed():
mgr.bus = MessageBus()
mgr.channels = {"streamed": _StreamedChannel(fake_config, mgr.bus)}
mgr._dispatch_task = None
+ mgr._outbound_sends = asyncio.Semaphore(32)
msg = outbound_message_for_event(
channel="streamed",
@@ -3309,6 +3314,7 @@ async def test_send_with_retry_propagates_cancelled_error():
mgr.bus = MessageBus()
mgr.channels = {"cancelling": _CancellingChannel(fake_config, mgr.bus)}
mgr._dispatch_task = None
+ mgr._outbound_sends = asyncio.Semaphore(32)
msg = OutboundMessage(channel="cancelling", chat_id="123", content="test")
@@ -3346,6 +3352,7 @@ async def test_send_with_retry_propagates_cancelled_error_during_sleep():
mgr.bus = MessageBus()
mgr.channels = {"failing": _FailingChannel(fake_config, mgr.bus)}
mgr._dispatch_task = None
+ mgr._outbound_sends = asyncio.Semaphore(32)
msg = OutboundMessage(channel="failing", chat_id="123", content="test")
@@ -3837,6 +3844,7 @@ async def test_restart_notice_retries_until_running_channel_accepts_delivery():
mgr = ChannelManager.__new__(ChannelManager)
mgr.config = fake_config
mgr.bus = MessageBus()
+ mgr._outbound_sends = asyncio.Semaphore(32)
channel = _EventuallyDeliverableChannel(fake_config, mgr.bus)
channel._running = True
mgr.channels = {"discord": channel}
diff --git a/tests/channels/test_send_retry_policy.py b/tests/channels/test_send_retry_policy.py
new file mode 100644
index 000000000..9847eaf82
--- /dev/null
+++ b/tests/channels/test_send_retry_policy.py
@@ -0,0 +1,127 @@
+"""Reachable manager/adapter ownership contracts for outbound retries."""
+
+import asyncio
+import random
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+# Telegram is an optional channel dependency, as in its adapter-owned suite.
+pytest.importorskip("telegram")
+from telegram.error import NetworkError, RetryAfter, TimedOut
+
+from nanobot.bus.delivery import ChannelNotReadyError
+from nanobot.bus.events import OutboundMessage
+from nanobot.bus.queue import MessageBus
+from nanobot.channels import manager as manager_module
+from nanobot.channels.manager import ChannelManager
+from nanobot.channels.telegram import runtime as telegram_module
+from nanobot.channels.telegram.runtime import (
+ TELEGRAM_MAX_MESSAGE_LEN,
+ TelegramChannel,
+ _split_telegram_markdown,
+)
+from nanobot.config.schema import Config
+
+
+def make_manager(tmp_path):
+ config = Config.model_validate({"channels": {"websocket": {"enabled": False}}})
+ return ChannelManager(config, MessageBus(), config_path=tmp_path / "config.json")
+
+
+def telegram_channel(manager):
+ channel = TelegramChannel({"token": "test", "rich_messages": False}, manager.bus)
+ channel._app = SimpleNamespace(bot=SimpleNamespace(send_message=AsyncMock()))
+ channel._app_ready.set()
+ return channel
+
+
+async def test_telegram_exhausted_second_chunk_never_replays_successful_prefix(tmp_path, monkeypatch):
+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (0,))
+ monkeypatch.setattr(telegram_module, "_SEND_RETRY_BASE_DELAY", 0)
+ manager = make_manager(tmp_path)
+ channel = telegram_channel(manager)
+ text = "a" * TELEGRAM_MAX_MESSAGE_LEN + "\n" + "b" * 20
+ chunks = _split_telegram_markdown(text, TELEGRAM_MAX_MESSAGE_LEN)
+ assert len(chunks) == 2
+ submitted = []
+
+ async def send_message(**kwargs):
+ submitted.append(kwargs["text"])
+ if kwargs["text"] != chunks[0]:
+ raise TimedOut("second part exhausted")
+ return SimpleNamespace(message_id=1)
+
+ channel._app.bot.send_message.side_effect = send_message
+ await manager._send_with_retry(channel, OutboundMessage("telegram", "1", text))
+ assert submitted == [chunks[0], chunks[1], chunks[1], chunks[1]]
+
+
[email protected](("error", "requests"), [
+ (TimedOut("timeout"), 3),
+ (RetryAfter(0), 3),
+ (NetworkError("ambiguous transport error"), 1),
+ (RuntimeError("unknown send result"), 1),
+])
+async def test_telegram_manager_does_not_multiply_part_retry_budget(tmp_path, monkeypatch, error, requests):
+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (0,))
+ monkeypatch.setattr(telegram_module, "_SEND_RETRY_BASE_DELAY", 0)
+ manager = make_manager(tmp_path)
+ channel = telegram_channel(manager)
+ channel._app.bot.send_message.side_effect = error
+ await manager._send_with_retry(channel, OutboundMessage("telegram", "1", "result"))
+ assert channel._app.bot.send_message.await_count == requests
+
+
+async def test_telegram_manager_retries_only_pre_submit_not_ready(tmp_path, monkeypatch):
+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (0,))
+ manager = make_manager(tmp_path)
+ channel = telegram_channel(manager)
+ channel._wait_for_app = AsyncMock(side_effect=[None, channel._app])
+ await manager._send_with_retry(channel, OutboundMessage("telegram", "1", "result"))
+ assert channel._wait_for_app.await_count == 2
+ channel._app.bot.send_message.assert_awaited_once()
+
+
+async def test_manager_equal_jitter_uses_capped_retry_delays(tmp_path, monkeypatch):
+ manager = make_manager(tmp_path)
+ manager.config.channels.send_max_retries = 6
+ channel = SimpleNamespace(send=AsyncMock(side_effect=OSError("offline")),
+ should_retry_send_error=Mock(return_value=True))
+ uniform = Mock(side_effect=lambda lower, upper: (lower + upper) / 2)
+ monkeypatch.setattr(random, "uniform", uniform)
+ sleep = AsyncMock()
+ monkeypatch.setattr(asyncio, "sleep", sleep)
+ await manager._send_with_retry(channel, OutboundMessage("test", "1", "result"))
+ assert [call.args for call in uniform.call_args_list] == [(0.5, 1), (1.0, 2), (2.0, 4), (2.0, 4), (2.0, 4)]
+ assert [call.args[0] for call in sleep.await_args_list] == [0.75, 1.5, 3, 3, 3]
+
+
+async def test_manager_deadline_clamps_jitter_and_keeps_pre_submit_retry(tmp_path, monkeypatch):
+ manager = make_manager(tmp_path)
+ manager.config.channels.send_max_retries = 1
+ channel = SimpleNamespace(send=AsyncMock(side_effect=[ChannelNotReadyError("starting"), None]),
+ should_retry_send_error=Mock(return_value=True))
+ monkeypatch.setattr(manager_module, "_SEND_RETRY_DELAYS", (4,))
+ monkeypatch.setattr(random, "uniform", lambda lower, upper: upper)
+ sleep = AsyncMock()
+ monkeypatch.setattr(asyncio, "sleep", sleep)
+ await manager._send_with_retry(channel, OutboundMessage("test", "1", "result"),
+ deadline=asyncio.get_running_loop().time() + 2)
+ assert channel.send.await_count == 2
+ assert 0 < sleep.await_args.args[0] <= 2
+
+
+async def test_telegram_restart_notice_deadline_never_replays_unknown_result(tmp_path):
+ from nanobot.utils.restart import RestartNotice
+
+ manager = make_manager(tmp_path)
+ channel = telegram_channel(manager)
+ channel._running = True
+ channel._app.bot.send_message.side_effect = NetworkError("unknown delivery")
+ manager.channels["telegram"] = channel
+ await manager._send_restart_notice_when_started(
+ RestartNotice(channel="telegram", chat_id="1", started_at_raw=""), timeout_s=0.1,
+ )
+ channel._app.bot.send_message.assert_awaited_once()
diff --git a/tests/providers/test_openai_codex_provider.py b/tests/providers/test_openai_codex_provider.py
index 7ad250ea6..636eeaac5 100644
--- a/tests/providers/test_openai_codex_provider.py
+++ b/tests/providers/test_openai_codex_provider.py
@@ -698,7 +698,8 @@ async def test_codex_retry_uses_structured_transient_error_metadata(monkeypatch,
assert response.content == "ok"
assert calls == 2
- assert delays == [1]
+ assert len(delays) == 1
+ assert 0.5 <= delays[0] <= 1.0
@pytest.mark.asyncio
diff --git a/tests/providers/test_provider_retry.py b/tests/providers/test_provider_retry.py
index 8799793ec..3d939cd15 100644
--- a/tests/providers/test_provider_retry.py
+++ b/tests/providers/test_provider_retry.py
@@ -1,5 +1,6 @@
import asyncio
import copy
+import random
import pytest
@@ -14,6 +15,11 @@ from nanobot.providers.base import (
)
+def assert_local_backoff(delays: list[float], ceilings: tuple[float, ...]) -> None:
+ assert len(delays) == len(ceilings)
+ assert all(ceiling / 2 <= delay <= ceiling for delay, ceiling in zip(delays, ceilings))
+
+
class ScriptedProvider(LLMProvider):
def __init__(self, responses):
super().__init__(provider_name="scripted")
@@ -64,9 +70,9 @@ async def test_chat_with_retry_retries_transient_error_then_succeeds(monkeypatch
LLMResponse(content="429 rate limit", finish_reason="error"),
LLMResponse(content="ok"),
])
- delays: list[int] = []
+ delays: list[float] = []
- async def _fake_sleep(delay: int) -> None:
+ async def _fake_sleep(delay: float) -> None:
delays.append(delay)
monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)
@@ -76,7 +82,7 @@ async def test_chat_with_retry_retries_transient_error_then_succeeds(monkeypatch
assert response.finish_reason == "stop"
assert response.content == "ok"
assert provider.calls == 2
- assert delays == [1]
+ assert_local_backoff(delays, (1,))
@pytest.mark.asyncio
@@ -156,9 +162,9 @@ async def test_chat_with_retry_does_not_retry_non_transient_error(monkeypatch) -
provider = ScriptedProvider([
LLMResponse(content="401 unauthorized", finish_reason="error"),
])
- delays: list[int] = []
+ delays: list[float] = []
- async def _fake_sleep(delay: int) -> None:
+ async def _fake_sleep(delay: float) -> None:
delays.append(delay)
monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)
@@ -171,25 +177,37 @@ async def test_chat_with_retry_does_not_retry_non_transient_error(monkeypatch) -
@pytest.mark.asyncio
-async def test_chat_with_retry_returns_final_error_after_retries(monkeypatch) -> None:
[email protected]("fraction", [0.0, 1.0])
+async def test_chat_with_retry_returns_final_error_after_retries(monkeypatch, fraction) -> None:
provider = ScriptedProvider([
LLMResponse(content="429 rate limit a", finish_reason="error"),
LLMResponse(content="429 rate limit b", finish_reason="error"),
LLMResponse(content="429 rate limit c", finish_reason="error"),
LLMResponse(content="503 final server error", finish_reason="error"),
])
- delays: list[int] = []
+ delays: list[float] = []
+ statuses: list[RetryStatusEvent] = []
+ monkeypatch.setattr(random, "uniform", lambda low, high: low + (high - low) * fraction)
+ monkeypatch.setattr("nanobot.providers.base.time.time", lambda: 1000.0)
- async def _fake_sleep(delay: int) -> None:
+ async def _fake_sleep(delay: float) -> None:
delays.append(delay)
- monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)
+ async def status(event: RetryStatusEvent) -> None:
+ statuses.append(event)
- response = await provider.chat_with_retry(messages=[{"role": "user", "content": "hello"}])
+ monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)
+ response = await provider.chat_with_retry(
+ messages=[{"role": "user", "content": "hello"}], on_retry_status=status,
+ )
assert response.content == "503 final server error"
assert provider.calls == 4
- assert delays == [1, 2, 4]
+ assert delays == [base * (0.5 + fraction / 2) for base in (1, 2, 4)]
+ assert [event.next_retry_at for event in statuses if event.state == "waiting"] == [
+ 1000.0 + delay for delay in delays
+ ]
+ assert statuses[-1].state == "exhausted"
@pytest.mark.asyncio
@@ -207,7 +225,7 @@ async def test_chat_with_retry_emits_terminal_progress_when_standard_retries_exh
progress: list[str] = []
statuses: list[RetryStatusEvent] = []
- async def _fake_sleep(delay: int) -> None:
+ async def _fake_sleep(delay: float) -> None:
return None
async def _progress(msg: str) -> None:
@@ -245,7 +263,7 @@ async def test_chat_with_retry_routes_terminal_progress_to_explicit_callback(mon
retry_progress: list[str] = []
terminal_progress: list[str] = []
- async def _fake_sleep(delay: int) -> None:
+ async def _fake_sleep(delay: float) -> None:
return None
async def _retry_progress(msg: str) -> None:
@@ -284,9 +302,9 @@ async def test_chat_stream_with_retry_does_not_retry_after_emitting_content(monk
LLMResponse(content="ok"),
])
deltas: list[str] = []
- delays: list[int] = []
+ delays: list[float] = []
- async def _fake_sleep(delay: int) -> None:
+ async def _fake_sleep(delay: float) -> None:
delays.append(delay)
async def _on_delta(delta: str) -> None:
@@ -318,9 +336,9 @@ async def test_chat_stream_with_retry_retries_timeout_after_emitting_content(mon
LLMResponse(content="full retry response"),
])
deltas: list[str] = []
- delays: list[int] = []
+ delays: list[float] = []
- async def _fake_sleep(delay: int) -> None:
+ async def _fake_sleep(delay: float) -> None:
delays.append(delay)
async def _on_delta(delta: str) -> None:
@@ -337,7 +355,7 @@ async def test_chat_stream_with_retry_retries_timeout_after_emitting_content(mon
assert response.finish_reason == "stop"
assert provider.calls == 2
assert deltas == ["partial"]
- assert delays == [1]
+ assert_local_backoff(delays, (1,))
assert provider.last_kwargs.get("on_content_delta") is None
@@ -356,9 +374,9 @@ async def test_chat_stream_with_retry_retries_timeout_in_new_stream_segment(
provider = ScriptedProvider([first, second])
deltas: list[str] = []
recoveries: list[str] = []
- delays: list[int] = []
+ delays: list[float] = []
- async def _fake_sleep(delay: int) -> None:
+ async def _fake_sleep(delay: float) -> None:
delays.append(delay)
async def _on_delta(delta: str) -> None:
@@ -380,7 +398,7 @@ async def test_chat_stream_with_retry_retries_timeout_in_new_stream_segment(
assert provider.calls == 2
assert deltas == ["partial", "full retry response"]
assert recoveries == ["recover"]
- assert delays == [1]
+ assert_local_backoff(delays, (1,))
assert provider.last_kwargs.get("on_content_delta") is not None
@@ -633,7 +651,8 @@ async def test_image_fallback_without_meta_uses_default_placeholder() -> None:
@pytest.mark.asyncio
-async def test_chat_with_retry_uses_retry_after_and_emits_wait_progress(monkeypatch) -> None:
[email protected]("retry_mode", ["standard", "persistent"])
+async def test_chat_with_retry_uses_retry_after_and_emits_wait_progress(monkeypatch, retry_mode) -> None:
provider = ScriptedProvider([
LLMResponse(content="429 rate limit, retry after 7s", finish_reason="error"),
LLMResponse(content="ok"),
@@ -641,17 +660,21 @@ async def test_chat_with_retry_uses_retry_after_and_emits_wait_progress(monkeypa
delays: list[float] = []
progress: list[str] = []
+ def unexpected_draw(*args):
+ pytest.fail("A provider-directed Retry-After must not be jittered")
+
async def _fake_sleep(delay: float) -> None:
delays.append(delay)
async def _progress(msg: str) -> None:
progress.append(msg)
+ monkeypatch.setattr(random, "uniform", unexpected_draw)
monkeypatch.setattr("nanobot.providers.base.asyncio.sleep", _fake_sleep)
response = await provider.chat_with_retry(
messages=[{"role": "user", "content": "hello"}],
- on_retry_wait=_progress,
+ on_retry_wait=_progress, retry_mode=retry_mode,
)
assert response.content == "ok"
@@ -744,7 +767,7 @@ async def test_chat_with_retry_retries_structured_status_code_without_keyword(mo
assert response.content == "ok"
assert provider.calls == 2
- assert delays == [1]
+ assert_local_backoff(delays, (1,))
@pytest.mark.asyncio
@@ -821,7 +844,7 @@ async def test_chat_with_retry_retries_structured_timeout_kind(monkeypatch) -> N
assert response.content == "ok"
assert provider.calls == 2
- assert delays == [1]
+ assert_local_backoff(delays, (1,))
@pytest.mark.asyncio
@@ -891,7 +914,7 @@ async def test_persistent_retry_aborts_after_ten_identical_transient_errors(monk
assert response.finish_reason == "error"
assert response.content == "429 rate limit"
assert provider.calls == 10
- assert delays == [1, 2, 4, 4, 4, 4, 4, 4, 4]
+ assert_local_backoff(delays, (1, 2, 4, 4, 4, 4, 4, 4, 4))
@pytest.mark.asyncio
@@ -963,7 +986,7 @@ async def test_chat_with_retry_retries_zhipu_1302_rate_limit(monkeypatch) -> Non
assert response.content == "ok"
assert provider.calls == 2
- assert delays == [1]
+ assert_local_backoff(delays, (1,))
@pytest.mark.asyncio
@@ -989,7 +1012,7 @@ async def test_chat_with_retry_retries_zhipu_1302_with_429_status(monkeypatch) -
assert response.content == "ok"
assert provider.calls == 2
- assert delays == [1]
+ assert_local_backoff(delays, (1,))
@pytest.mark.asyncio
@@ -1006,3 +1029,60 @@ async def test_chat_stream_with_retry_normalizes_explicit_none_max_tokens() -> N
assert response.content == "ok"
assert provider.last_kwargs["max_tokens"] == 4096
assert provider.last_kwargs["temperature"] == 0.7
+
+
+async def test_concurrent_requests_keep_retry_status_and_attempts_local(monkeypatch) -> None:
+ ready = asyncio.Event()
+ calls = {"a": 0, "b": 0}
+ statuses = {"a": [], "b": []}
+
+ class ConcurrentProvider(ScriptedProvider):
+ async def chat(self, messages, **kwargs):
+ key = messages[-1]["content"]
+ calls[key] += 1
+ if calls[key] == 1:
+ if all(calls.values()):
+ ready.set()
+ await ready.wait()
+ return LLMResponse(content="connection failed", finish_reason="error")
+ return LLMResponse(content=key)
+
+ provider = ConcurrentProvider([])
+ monkeypatch.setattr(provider, "_CHAT_RETRY_DELAYS", (0.1,))
+
+ async def request(key):
+ async def status(event):
+ statuses[key].append((event.state, event.attempt, event.max_attempts))
+
+ return await provider.chat_with_retry(
+ messages=[{"role": "user", "content": key}], on_retry_status=status,
+ )
+
+ results = await asyncio.wait_for(asyncio.gather(request("a"), request("b")), 2)
+ assert [result.content for result in results] == ["a", "b"]
+ assert calls == {"a": 2, "b": 2}
+ assert statuses == {key: [("waiting", 1, 2), ("recovered", 2, 2)] for key in calls}
+
+
+async def test_cancellation_interrupts_retry_wait_without_another_call() -> None:
+ provider = ScriptedProvider([LLMResponse(
+ content="rate limited", finish_reason="error", error_status_code=429, error_retry_after_s=120,
+ )])
+ waiting = asyncio.Event()
+ statuses = []
+
+ async def status(event):
+ statuses.append((event.state, event.attempt))
+ waiting.set()
+
+ task = asyncio.create_task(provider.chat_with_retry(messages=[], on_retry_status=status))
+ try:
+ await asyncio.wait_for(waiting.wait(), 2)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ finally:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ assert provider.calls == 1
+ assert statuses == [("waiting", 1)]
diff --git a/tests/providers/test_provider_sdk_retry_defaults.py b/tests/providers/test_provider_sdk_retry_defaults.py
index bf4f10bda..b5fb86a19 100644
--- a/tests/providers/test_provider_sdk_retry_defaults.py
+++ b/tests/providers/test_provider_sdk_retry_defaults.py
@@ -1,7 +1,10 @@
from unittest.mock import patch
+import pytest
+
from nanobot.providers.anthropic_provider import AnthropicProvider
from nanobot.providers.azure_openai_provider import AzureOpenAIProvider
+from nanobot.providers.bedrock_provider import BedrockProvider
from nanobot.providers.openai_compat_provider import OpenAICompatProvider
@@ -44,3 +47,13 @@ def test_azure_openai_disables_sdk_retries_by_default() -> None:
kwargs = mock_client.call_args.kwargs
assert kwargs["max_retries"] == 0
+
+
+def test_bedrock_disables_sdk_retries_even_with_environment_attempts(monkeypatch) -> None:
+ pytest.importorskip("boto3")
+ monkeypatch.setenv("AWS_MAX_ATTEMPTS", "10")
+ with patch("boto3.Session") as session:
+ BedrockProvider(api_base="http://127.0.0.1:1", region="us-east-1")
+
+ config = session.return_value.client.call_args.kwargs["config"]
+ assert config.retries["total_max_attempts"] == 1
diff --git a/tests/providers/test_stepfun_asr.py b/tests/providers/test_stepfun_asr.py
index 4074f0a7e..867cb26f6 100644
--- a/tests/providers/test_stepfun_asr.py
+++ b/tests/providers/test_stepfun_asr.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import asyncio
+import base64
import json
from pathlib import Path
from typing import Any
@@ -132,11 +134,14 @@ async def test_sse_error_event(audio_file: Path) -> None:
provider = StepFunTranscriptionProvider(api_key="sk-test")
stream_cm = _make_stream_cm(200, lines)
+ sleep = AsyncMock()
- with patch("httpx.AsyncClient.stream", stream_cm):
+ with patch("httpx.AsyncClient.stream", stream_cm), patch("asyncio.sleep", sleep):
result = await provider.transcribe(audio_file)
assert result == ""
+ assert stream_cm.call_count == 1
+ sleep.assert_not_awaited()
@pytest.mark.asyncio
@@ -186,36 +191,78 @@ async def test_sse_malformed_json_skipped(audio_file: Path) -> None:
@pytest.mark.asyncio
-async def test_retries_on_503_then_succeeds(audio_file: Path) -> None:
- """Transient 503 is retried, then a successful SSE stream yields text."""
- success_lines = [
- f"data: {json.dumps({'type': 'transcript.text.done', 'session_id': 's1', 'text': 'ok'})}",
- ]
- # First call: 503 (FailingResponse), second call: success (FakeResponse with lines)
- stream_cm = _make_stream_cm_sequence([503, success_lines])
-
- provider = StepFunTranscriptionProvider(api_key="sk-test")
- with patch("httpx.AsyncClient.stream", stream_cm), patch(
- "asyncio.sleep", AsyncMock()
- ):
- result = await provider.transcribe(audio_file)
-
- assert result == "ok"
+ "scenario,expected",
+ [
+ ([(503, [], None), (200, [], None), (200, ["data: []"], None),
+ (200, ['data: {"type":"transcript.text.done","text":"ok"}'], None)], "ok"),
+ ([(200, [], httpx.ReadError("lost")),
+ (200, ['data: {"type":"transcript.text.done","text":"ok"}'], None)], "ok"),
+ ([(200, [], None)] * 4, ""),
+ ([(200, ["data: []"], None)] * 4, ""),
+ ],
+ ids=["mixed-budget", "read-error", "missing-final-exhaustion", "parser-exhaustion"],
+)
+async def test_sse_closes_each_attempt_before_backoff(monkeypatch, audio_file, scenario, expected):
+ original_audio = audio_file.read_bytes()
+ remaining = iter(scenario)
+ streams, bodies, waits = [], [], []
+
+ class Stream(httpx.AsyncByteStream):
+ closed = False
+
+ def __init__(self, lines, error):
+ self.lines, self.error = lines, error
+
+ async def __aiter__(self):
+ for line in self.lines:
+ yield (line + "\n").encode()
+ if self.error is not None:
+ raise self.error
+
+ async def aclose(self):
+ self.closed = True
+
+ def handle(request):
+ bodies.append(json.loads(request.content))
+ status, lines, error = next(remaining)
+ stream = Stream(lines, error)
+ streams.append(stream)
+ return httpx.Response(status, stream=stream)
+
+ client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+
+ async def sleep(delay):
+ assert streams[-1].closed
+ waits.append(delay)
+ audio_file.write_bytes(b"changed-after-request")
+
+ monkeypatch.setattr("nanobot.providers.transcription.httpx.AsyncClient", lambda: client)
+ monkeypatch.setattr("nanobot.providers.transcription.asyncio.sleep", sleep)
+ result = await StepFunTranscriptionProvider(api_key="sk-test", language="zh").transcribe(audio_file)
+ assert result == expected
+ assert len(bodies) == len(scenario)
+ assert bodies == [bodies[0]] * len(scenario)
+ assert base64.b64decode(bodies[0]["audio"]["data"]) == original_audio
+ assert bodies[0]["audio"]["input"]["transcription"]["language"] == "zh"
+ assert waits == [1.0, 2.0, 4.0][:len(scenario) - 1]
+ assert all(stream.closed for stream in streams)
+ assert client.is_closed
@pytest.mark.asyncio
async def test_gives_up_after_max_retries(audio_file: Path) -> None:
"""Persistent 503 returns "" after all retries exhausted."""
- attempts: list[list[str] | int] = [503, 503, 503, 503] # 4 failing HTTP responses
- stream_cm = _make_stream_cm_sequence(attempts)
+ stream_cm = _make_stream_cm(503, [])
provider = StepFunTranscriptionProvider(api_key="sk-test")
- with patch("httpx.AsyncClient.stream", stream_cm), patch(
- "asyncio.sleep", AsyncMock()
- ):
+ sleep = AsyncMock()
+ with patch("httpx.AsyncClient.stream", stream_cm), patch("asyncio.sleep", sleep):
result = await provider.transcribe(audio_file)
assert result == ""
+ assert stream_cm.call_count == 4
+ assert [call.args[0] for call in sleep.await_args_list] == [1.0, 2.0, 4.0]
@pytest.mark.asyncio
@@ -229,18 +276,19 @@ async def test_sse_empty_text_done_returns_empty(audio_file: Path) -> None:
provider = StepFunTranscriptionProvider(api_key="sk-test")
stream_cm = _make_stream_cm(200, lines)
- with patch("httpx.AsyncClient.stream", stream_cm), patch(
- "asyncio.sleep", AsyncMock()
- ):
+ sleep = AsyncMock()
+ with patch("httpx.AsyncClient.stream", stream_cm), patch("asyncio.sleep", sleep):
result = await provider.transcribe(audio_file)
assert result == ""
+ assert stream_cm.call_count == 1
+ sleep.assert_not_awaited()
@pytest.mark.asyncio
-async def test_401_returns_empty_without_retry(audio_file: Path) -> None:
- """401 is not retryable; bad credentials should fail immediately."""
- stream_cm = _make_stream_cm(401, [])
[email protected]("status", [401, 501])
+async def test_non_retryable_status_returns_empty(audio_file: Path, status: int) -> None:
+ stream_cm = _make_stream_cm(status, [])
sleep = AsyncMock()
provider = StepFunTranscriptionProvider(api_key="sk-test")
@@ -258,32 +306,9 @@ async def test_retries_on_connect_error(audio_file: Path) -> None:
success_lines = [
f"data: {json.dumps({'type': 'transcript.text.done', 'session_id': 's1', 'text': 'ok'})}",
]
- call_count = [0]
-
- class FakeResponse:
- """Serves as both the async context manager returned by stream()
- and the response object bound in `async with ... as resp`."""
- status_code = 200
- reason_phrase = "OK"
-
- async def __aenter__(self) -> "FakeResponse":
- return self
-
- async def __aexit__(self, *exc: object) -> None:
- pass
-
- async def aiter_lines(self) -> Any:
- for line in success_lines:
- yield line
-
- def raise_for_status(self) -> None:
- pass
-
- def fake_stream(method: str, url: str, *args: object, **kwargs: object) -> FakeResponse:
- call_count[0] += 1
- if call_count[0] == 1:
- raise httpx.ConnectError("boom")
- return FakeResponse()
+ fake_stream = MagicMock(side_effect=[
+ httpx.ConnectError("boom"), _make_stream_cm(200, success_lines).return_value,
+ ])
provider = StepFunTranscriptionProvider(api_key="sk-test")
with patch("httpx.AsyncClient.stream", fake_stream), patch(
@@ -292,7 +317,82 @@ async def test_retries_on_connect_error(audio_file: Path) -> None:
result = await provider.transcribe(audio_file)
assert result == "ok"
- assert call_count[0] == 2
+ assert fake_stream.call_count == 2
+
+
[email protected]("cancel_backoff", [False, True], ids=["recover", "cancel-backoff"])
+async def test_unread_503_socket_closes_before_backoff(monkeypatch, audio_file, cancel_backoff):
+ first_closed, entered, blocked = asyncio.Event(), asyncio.Event(), asyncio.Event()
+ bodies, responses, waits = [], [], []
+ handlers = set()
+
+ async def handle(reader, writer):
+ handler = asyncio.current_task()
+ handlers.add(handler)
+ try:
+ head = await reader.readuntil(b"\r\n\r\n")
+ headers = dict(line.split(b":", 1) for line in head.split(b"\r\n")[1:] if b":" in line)
+ bodies.append(json.loads(await reader.readexactly(int(headers[b"Content-Length"]))))
+ first = len(bodies) == 1
+ if first:
+ # The body never finishes: the retry must close it without reading EOF.
+ writer.write(b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 1000000\r\n\r\nunread")
+ else:
+ payload = b'data: {"type":"transcript.text.done","text":"socket-ok"}\n\n'
+ writer.write(b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nContent-Length: "
+ + str(len(payload)).encode() + b"\r\n\r\n" + payload)
+ await writer.drain()
+ await reader.read()
+ if first:
+ first_closed.set()
+ finally:
+ writer.close()
+ await writer.wait_closed()
+ handlers.discard(handler)
+
+ async def response_received(response):
+ responses.append(response)
+
+ async def sleep(delay):
+ waits.append(delay)
+ assert responses[-1].is_closed
+ await asyncio.wait_for(first_closed.wait(), timeout=2)
+ entered.set()
+ if cancel_backoff:
+ await blocked.wait()
+
+ server = await asyncio.start_server(handle, "127.0.0.1", 0)
+ port = server.sockets[0].getsockname()[1]
+ client = httpx.AsyncClient(trust_env=False, event_hooks={"response": [response_received]})
+ monkeypatch.setattr("nanobot.providers.transcription.httpx.AsyncClient", lambda: client)
+ monkeypatch.setattr("nanobot.providers.transcription.asyncio.sleep", sleep)
+ task = asyncio.create_task(StepFunTranscriptionProvider(
+ api_key="local", api_base=f"http://127.0.0.1:{port}",
+ ).transcribe(audio_file))
+ try:
+ await asyncio.wait_for(entered.wait(), timeout=5)
+ if cancel_backoff:
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert len(bodies) == 1
+ else:
+ assert await asyncio.wait_for(task, timeout=5) == "socket-ok"
+ assert len(bodies) == 2
+ assert bodies[0] == bodies[1]
+ assert waits == [1.0]
+ assert client.is_closed
+ finally:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ await client.aclose()
+ server.close()
+ await server.wait_closed()
+ pending = list(handlers)
+ for handler in pending:
+ handler.cancel()
+ await asyncio.wait_for(asyncio.gather(*pending, return_exceptions=True), timeout=5)
# ---------------------------------------------------------------------------
@@ -362,66 +462,3 @@ def _make_stream_cm(status: int, lines: list[str]) -> MagicMock:
cm = MagicMock()
cm.return_value = FakeResponse()
return cm
-
-
-def _make_stream_cm_sequence(statuses: list[str | int]) -> MagicMock:
- """Build a stream mock that fails with HTTP status ints, then succeeds with SSE lines.
-
- Entries in *statuses* that are ints produce a stream that raises HTTPStatusError
- after `raise_for_status()`. The final entry (a list of SSE lines) succeeds.
- """
- remaining = list(statuses)
-
- class FakeResponse:
- def __init__(self, lines: list[str]) -> None:
- self._lines = lines
- self.status_code = 200
- self.reason_phrase = "OK"
-
- async def __aenter__(self) -> "FakeResponse":
- return self
-
- async def __aexit__(self, *exc: object) -> None:
- pass
-
- async def aiter_lines(self) -> Any:
- for line in self._lines:
- yield line
-
- def raise_for_status(self) -> None:
- pass
-
- class FailingResponse:
- def __init__(self, status: int) -> None:
- self.status_code = status
- self.reason_phrase = "Error"
-
- async def __aenter__(self) -> "FailingResponse":
- return self
-
- async def __aexit__(self, *exc: object) -> None:
- pass
-
- async def aiter_lines(self) -> Any:
- yield ""
- return
-
- def raise_for_status(self) -> None:
- raise httpx.HTTPStatusError(
- f"HTTP {self.status_code}",
- request=httpx.Request("POST", "https://example.test"),
- response=httpx.Response(self.status_code),
- )
-
- call_count = [0]
-
- def _next(method: str, url: str, **kwargs: object) -> Any:
- idx = min(call_count[0], len(remaining) - 1)
- entry = remaining[idx]
- call_count[0] += 1
- if isinstance(entry, int):
- return FailingResponse(entry)
- return FakeResponse(entry)
-
- cm = MagicMock(side_effect=_next)
- return cm
diff --git a/tests/providers/test_transcription.py b/tests/providers/test_transcription.py
index 99ae1258a..d1ad4d744 100644
--- a/tests/providers/test_transcription.py
+++ b/tests/providers/test_transcription.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import asyncio
import base64
import os
from pathlib import Path
@@ -26,6 +27,7 @@ from nanobot.providers.transcription import (
GroqTranscriptionProvider,
OpenAITranscriptionProvider,
OpenRouterTranscriptionProvider,
+ StepFunTranscriptionProvider,
XiaomiMiMoTranscriptionProvider,
_audio_format,
_resolve_chat_completions_url,
@@ -419,40 +421,30 @@ def test_resolver_keeps_enabled_and_limits_on_effective_config() -> None:
@pytest.mark.asyncio
-async def test_openai_does_not_retry_on_auth_error(audio_file: Path) -> None:
- """401 is the user's misconfiguration — retrying wastes time and rate-limit quota."""
[email protected]("outcome", [_response(401), _response(501), httpx.LocalProtocolError("bad")])
+async def test_openai_does_not_retry_terminal_errors(audio_file: Path, outcome) -> None:
provider = OpenAITranscriptionProvider(api_key="sk-test")
- post = AsyncMock(return_value=_response(401, {"error": {"message": "bad key"}}))
- with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", AsyncMock()):
+ post = AsyncMock(side_effect=[outcome])
+ sleep = AsyncMock()
+ with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", sleep):
result = await provider.transcribe(audio_file)
assert result == ""
assert post.await_count == 1
+ sleep.assert_not_awaited()
@pytest.mark.asyncio
-async def test_openai_gives_up_after_max_attempts(audio_file: Path) -> None:
- """Persistent 503 returns "" after the final retry — never hangs."""
[email protected]("outcome", [_response(503), httpx.ConnectError("offline")])
+async def test_openai_gives_up_after_max_attempts(audio_file: Path, outcome) -> None:
provider = OpenAITranscriptionProvider(api_key="sk-test")
- post = AsyncMock(return_value=_response(503))
+ post = AsyncMock(side_effect=[outcome] * 4)
sleep = AsyncMock()
with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", sleep):
result = await provider.transcribe(audio_file)
assert result == ""
# 4 attempts total (initial + 3 retries) with 3 sleeps between them.
assert post.await_count == 4
- assert sleep.await_count == 3
-
-
-async def test_openai_backoff_grows_exponentially(audio_file: Path) -> None:
- """Verify the backoff schedule is exponential (1s, 2s, 4s)."""
- provider = OpenAITranscriptionProvider(api_key="sk-test")
- post = AsyncMock(return_value=_response(503))
- sleep = AsyncMock()
- with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", sleep):
- await provider.transcribe(audio_file)
- delays = [call.args[0] for call in sleep.await_args_list]
- assert delays == [1.0, 2.0, 4.0]
+ assert [call.args[0] for call in sleep.await_args_list] == [1.0, 2.0, 4.0]
# ---------------------------------------------------------------------------
@@ -590,16 +582,33 @@ async def test_provider_forwards_file_mime_type(tmp_path: Path) -> None:
@pytest.mark.asyncio
-async def test_language_survives_retry(audio_file: Path) -> None:
- """Regression: language must be present on every retry attempt, not just the first."""
- provider = OpenAITranscriptionProvider(api_key="sk-test", language="ja")
- post = AsyncMock(side_effect=[_response(503), _response(200, {"text": "konnichiwa"})])
- with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", AsyncMock()):
- result = await provider.transcribe(audio_file)
- assert result == "konnichiwa"
- assert post.await_count == 2
[email protected]("provider_cls", [OpenAITranscriptionProvider, OpenRouterTranscriptionProvider])
+async def test_json_retry_shares_budget_and_rebuilds_cached_audio(audio_file: Path, provider_cls):
+ original_audio = audio_file.read_bytes()
+ provider = provider_cls(api_key="sk-test", model="contract-model", language="ja")
+ post = AsyncMock(side_effect=[
+ httpx.ReadError("lost"), _response(503), httpx.WriteError("lost"),
+ _response(200, {"text": "konnichiwa"}),
+ ])
+ sleep = AsyncMock(side_effect=lambda _: audio_file.write_bytes(b"changed-after-request"))
+ with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", sleep):
+ assert await provider.transcribe(audio_file) == "konnichiwa"
+ assert post.await_count == 4
+ assert [call.args[0] for call in sleep.await_args_list] == [1.0, 2.0, 4.0]
+ field = "files" if provider_cls is OpenAITranscriptionProvider else "json"
+ bodies = [call.kwargs[field] for call in post.await_args_list]
+ assert len({id(body) for body in bodies}) == 4
+ expected = (
+ {"file": (audio_file.name, original_audio, "audio/ogg"),
+ "model": (None, "contract-model"), "language": (None, "ja")}
+ if field == "files" else
+ {"model": "contract-model", "language": "ja",
+ "input_audio": {"data": base64.b64encode(original_audio).decode(), "format": "ogg"}}
+ )
+ assert bodies == [expected] * 4
for call in post.await_args_list:
- assert call.kwargs["files"]["language"] == (None, "ja")
+ assert call.kwargs["headers"]["Authorization"] == "Bearer sk-test"
+ assert call.kwargs["timeout"] == 60.0
# ---------------------------------------------------------------------------
@@ -713,16 +722,6 @@ async def test_openrouter_omits_language_when_unset(audio_file: Path) -> None:
assert "language" not in post.await_args_list[0].kwargs["json"]
-async def test_openrouter_shares_retry_contract(audio_file: Path) -> None:
- """OpenRouter goes through the same retry helper: 503 retried, then 200."""
- provider = OpenRouterTranscriptionProvider(api_key="k", model="openai/whisper-1")
- post = AsyncMock(side_effect=[_response(503), _response(200, {"text": "recovered"})])
- with patch("httpx.AsyncClient.post", post), patch("asyncio.sleep", AsyncMock()):
- assert await provider.transcribe(audio_file) == "recovered"
- assert post.await_count == 2
-
-
def test_resolve_chat_completions_url_appends_path_to_base() -> None:
default = "https://api.xiaomimimo.com/v1/chat/completions"
assert _resolve_chat_completions_url(None, default) == default
@@ -971,6 +970,60 @@ async def test_retries_on_every_advertised_transient_exception(
assert post.await_count == 2
[email protected]("provider_cls", [OpenAITranscriptionProvider, StepFunTranscriptionProvider])
[email protected]("phase", ["request", "stream", "sleep"])
+async def test_transcription_cancellation_releases_client(monkeypatch, audio_file, provider_cls, phase):
+ entered, blocked = asyncio.Event(), asyncio.Event()
+ requests, streams, waits = [], [], []
+
+ class Stream(httpx.AsyncByteStream):
+ closed = False
+
+ async def __aiter__(self):
+ if phase == "stream":
+ entered.set()
+ await blocked.wait()
+ yield b""
+
+ async def aclose(self):
+ self.closed = True
+
+ async def handle(request):
+ requests.append(request)
+ if phase == "request":
+ entered.set()
+ await blocked.wait()
+ stream = Stream()
+ streams.append(stream)
+ return httpx.Response(200 if phase == "stream" else 503, stream=stream)
+
+ client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+
+ async def sleep(delay):
+ waits.append(delay)
+ assert streams[-1].closed
+ entered.set()
+ await blocked.wait()
+
+ monkeypatch.setattr("nanobot.providers.transcription.httpx.AsyncClient", lambda: client)
+ monkeypatch.setattr("nanobot.providers.transcription.asyncio.sleep", sleep)
+ task = asyncio.create_task(provider_cls(api_key="sk-test").transcribe(audio_file))
+ try:
+ await asyncio.wait_for(entered.wait(), timeout=5)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert len(requests) == 1
+ assert all(stream.closed for stream in streams)
+ assert client.is_closed
+ assert waits == ([1.0] if phase == "sleep" else [])
+ finally:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ await client.aclose()
+
+
# ---------------------------------------------------------------------------
# apiBase normalization (#3637): a chat-style base must not be POSTed verbatim
# ---------------------------------------------------------------------------
diff --git a/tests/tools/test_web_search_tool.py b/tests/tools/test_web_search_tool.py
index 4dc768b23..14fc1cbef 100644
--- a/tests/tools/test_web_search_tool.py
+++ b/tests/tools/test_web_search_tool.py
@@ -1,5 +1,8 @@
"""Tests for multi-provider web search."""
+import asyncio
+from contextlib import asynccontextmanager
+
import httpx
import pytest
@@ -97,7 +100,9 @@ async def test_brave_search_retries_rate_limit_once(monkeypatch):
async def mock_get(self, url, **kw):
calls["n"] += 1
if calls["n"] == 1:
- return _response(status=429, json={"error": "rate limit"})
+ response = _response(status=429, json={"error": "rate limit"})
+ response.headers["Retry-After"] = "3600"
+ return response
return _response(json={
"web": {"results": [{"title": "Recovered", "url": "https://example.com", "description": "ok"}]}
})
@@ -683,7 +688,9 @@ async def test_deepseek_search_retries_transient_errors_then_succeeds(
async def mock_post(self, url, **kw):
calls["n"] += 1
if calls["n"] == 1:
- return _response(status=status, json={"error": {"message": "transient"}})
+ response = _response(status=status, json={"error": {"message": "transient"}})
+ response.headers["Retry-After"] = "3600"
+ return response
return _response(json={
"results": [{"title": "Recovered", "url": "https://example.com/ok", "snippet": "ok"}]
})
@@ -1396,3 +1403,71 @@ async def test_olostep_package_missing_returns_install_hint(monkeypatch):
assert result == (
"Error: Olostep support is not installed. Run `nanobot plugins enable olostep`."
)
+
+
[email protected]("provider,outcome", [
+ ("brave", 500),
+ ("brave", httpx.ReadTimeout("slow")),
+ ("deepseek", httpx.ConnectError("offline")),
+])
+async def test_search_does_not_retry_unclassified_failures(monkeypatch, provider, outcome):
+ calls = []
+
+ async def request(*args, **kwargs):
+ calls.append(args)
+ if isinstance(outcome, Exception):
+ raise outcome
+ return _response(status=outcome)
+
+ method = "get" if provider == "brave" else "post"
+ monkeypatch.setattr(httpx.AsyncClient, method, request)
+ result = await _tool(provider=provider, api_key="test-key").execute(query="failure")
+ assert is_tool_error_result(result)
+ assert len(calls) == 1
+
+
[email protected]("provider", ["brave", "deepseek"])
[email protected]("phase", ["request", "backoff"])
+async def test_search_task_cancellation_closes_client_without_replay(monkeypatch, provider, phase):
+ entered, closed = asyncio.Event(), asyncio.Event()
+ calls = []
+ real_client = httpx.AsyncClient
+
+ async def block():
+ entered.set()
+ await asyncio.Event().wait()
+
+ async def handle(request):
+ calls.append(request)
+ if phase == "request":
+ await block()
+ return httpx.Response(429, json={})
+
+ @asynccontextmanager
+ async def client(**kwargs):
+ instance = real_client(transport=httpx.MockTransport(handle), **kwargs)
+ try:
+ async with instance:
+ yield instance
+ finally:
+ assert instance.is_closed
+ closed.set()
+
+ async def sleep(delay):
+ assert delay == 1.0
+ await block()
+
+ monkeypatch.setattr("nanobot.agent.tools.web.httpx.AsyncClient", client)
+ monkeypatch.setattr("nanobot.agent.tools.web.asyncio.sleep", sleep)
+ task = asyncio.create_task(_tool(provider=provider, api_key="test-key").execute(query="cancel"))
+ try:
+ await asyncio.wait_for(entered.wait(), timeout=2)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ finally:
+ if not task.done():
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ assert len(calls) == 1
+ assert closed.is_set()