Your lightweight open-source personal agent.
Python agent-frameworkai-agentai-agentschatbotchatopsdiscord-botllm-agentsllmslocal-llmmcpmodel-context-protocolmulti-agentopenai-compatibleopenclawpersonal-ai-assistantpythonself-hostedtelegram-bot-ai-assistantwebuiworkflow-automation
from __future__ import annotations
import asyncio
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from loguru import logger
from nanobot.config.schema import ModelPresetConfig
from nanobot.events import RetryStatusEvent
from nanobot.providers.base import (
LLMProvider,
LLMResponse,
ProviderCallContext,
ProviderConversationState,
)
from nanobot.providers.conversation_state import ProviderConversationStateController
from nanobot.providers.fallback_provider import FallbackModelSelection, FallbackProvider
from nanobot.providers.openai_responses import resolve_compact_threshold
def _make_response(
content: str = "ok",
finish_reason: str = "stop",
*,
error_kind: str | None = None,
error_status_code: int | None = None,
error_type: str | None = None,
error_code: str | None = None,
error_should_retry: bool | None = None,
) -> LLMResponse:
return LLMResponse(
content=content,
finish_reason=finish_reason,
error_kind=error_kind,
error_status_code=error_status_code,
error_type=error_type,
error_code=error_code,
error_should_retry=error_should_retry,
)
def _error_response(content: str = "api error") -> LLMResponse:
return _make_response(content, finish_reason="error", error_kind="server_error")
def _retryable_error(content: str = "") -> LLMResponse:
return _make_response(content, finish_reason="error", error_status_code=503)
def _fallback(
model: str,
provider: str = "custom",
*,
max_tokens: int = 8192,
context_window_tokens: int = 65_536,
temperature: float = 0.1,
reasoning_effort: str | None = None,
) -> ModelPresetConfig:
return ModelPresetConfig(
model=model,
provider=provider,
max_tokens=max_tokens,
context_window_tokens=context_window_tokens,
temperature=temperature,
reasoning_effort=reasoning_effort,
)
class _FakeProvider(LLMProvider):
"""Fake provider for testing."""
def __init__(
self,
name: str = "fake",
response: LLMResponse | None = None,
*,
responses: list[LLMResponse] | None = None,
):
super().__init__(provider_name=name)
self.name = name
self._response = response or _make_response()
self._responses = iter(responses) if responses is not None else None
self.chat_calls: list[dict[str, Any]] = []
self.chat_stream_calls: list[dict[str, Any]] = []
self.context_calls: list[ProviderCallContext | None] = []
self.resumable = False
self.compact = False
def _next_response(self) -> LLMResponse:
return next(self._responses) if self._responses is not None else self._response
def get_default_model(self) -> str:
return f"{self.name}/model"
async def chat(self, **kwargs: Any) -> LLMResponse:
self.chat_calls.append(dict(kwargs))
return self._next_response()
async def chat_stream(self, **kwargs: Any) -> LLMResponse:
self.chat_stream_calls.append(dict(kwargs))
response = self._next_response()
on_delta = kwargs.get("on_content_delta")
if on_delta and response.content:
await on_delta(response.content)
return response
async def chat_with_context(
self,
provider_context: ProviderCallContext | None = None,
**kwargs: Any,
) -> LLMResponse:
self.context_calls.append(provider_context)
return await self.chat(**kwargs)
def can_resume_conversation_state(
self,
state: ProviderConversationState,
model: str | None = None,
) -> bool:
_ = state, model
return self.resumable
def supports_native_compaction(self, model: str | None = None) -> bool:
_ = model
return self.compact
def test_fallback_models_default_empty() -> None:
from nanobot.config.schema import AgentDefaults
defaults = AgentDefaults()
assert defaults.fallback_models == []
def test_fallback_models_accept_preset_refs_and_inline_configs() -> None:
from nanobot.config.schema import Config, InlineFallbackConfig
config = Config.model_validate({
"agents": {
"defaults": {
"fallbackModels": [
"deep",
{
"provider": "openai",
"model": "gpt-4.1",
"maxTokens": 4096,
},
]
}
},
"modelPresets": {
"deep": {"provider": "anthropic", "model": "claude-opus-4-7"}
},
})
assert config.agents.defaults.fallback_models[0] == "deep"
assert config.agents.defaults.fallback_models[1] == InlineFallbackConfig(
provider="openai",
model="gpt-4.1",
max_tokens=4096,
)
async def test_fallback_preserves_the_operation_event_sink():
from nanobot.events import EventSink
async def observe(event):
pass
events = EventSink(observe)
primary = _FakeProvider("primary", _error_response())
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
fb = FallbackProvider(
primary=primary, fallback_presets=[_fallback("fallback-a")],
provider_factory=MagicMock(return_value=fallback),
)
result = await fb.chat_with_context(
messages=[{"role": "user", "content": "hello"}], model="primary",
provider_context=ProviderCallContext(events=events),
)
assert result.content == "fallback ok"
assert primary.context_calls[0].events is events
assert fallback.context_calls[0].events is events
def test_fallback_model_preset_ref_must_exist() -> None:
from nanobot.config.schema import Config
with pytest.raises(ValueError, match="fallback_models.*not found"):
Config.model_validate({
"agents": {"defaults": {"fallbackModels": ["missing"]}},
"modelPresets": {},
})
def test_provider_signature_tracks_fallback_presets_and_provider_config() -> None:
from nanobot.config.schema import Config
from nanobot.providers.factory import provider_signature
base = {
"agents": {
"defaults": {
"modelPreset": "fast",
"fallbackModels": ["deep"],
}
},
"modelPresets": {
"fast": {"model": "openai/gpt-4.1", "provider": "openai"},
"deep": {"model": "anthropic/claude-sonnet-4-6", "provider": "anthropic"},
},
"providers": {
"openai": {"apiKey": "primary-key"},
"anthropic": {"apiKey": "fallback-key"},
},
}
changed_fallback = {
**base,
"agents": {"defaults": {"modelPreset": "fast", "fallbackModels": ["backup"]}},
"modelPresets": {
**base["modelPresets"],
"backup": {"model": "deepseek/deepseek-chat", "provider": "deepseek"},
},
"providers": {
**base["providers"],
"deepseek": {"apiKey": "deepseek-key"},
},
}
changed_key = {
**base,
"providers": {
"openai": {"apiKey": "primary-key"},
"anthropic": {"apiKey": "new-fallback-key"},
},
}
signature = provider_signature(Config.model_validate(base))
assert signature != provider_signature(Config.model_validate(changed_fallback))
assert signature != provider_signature(Config.model_validate(changed_key))
def test_provider_snapshot_uses_smallest_fallback_context_window() -> None:
from nanobot.config.schema import Config
from nanobot.providers.factory import build_provider_snapshot
config = Config.model_validate({
"agents": {
"defaults": {
"modelPreset": "fast",
"fallbackModels": ["deep"],
}
},
"modelPresets": {
"fast": {
"model": "openai/gpt-4.1",
"provider": "openai",
"contextWindowTokens": 128000,
},
"deep": {
"model": "deepseek/deepseek-chat",
"provider": "deepseek",
"contextWindowTokens": 64000,
},
},
"providers": {
"openai": {"apiKey": "primary-key"},
"deepseek": {"apiKey": "fallback-key"},
},
})
with patch("nanobot.providers.openai_compat_provider.AsyncOpenAI"):
snapshot = build_provider_snapshot(config)
assert snapshot.context_window_tokens == 64000
assert isinstance(snapshot.provider, FallbackProvider)
assert snapshot.provider._primary_context_window_tokens == 128000
def test_factory_injects_configured_identity_into_primary_and_fallback_leaves() -> None:
from nanobot.config.schema import Config
from nanobot.providers.factory import build_provider_snapshot
config = Config.model_validate({
"agents": {
"defaults": {
"modelPreset": "primary",
"fallbackModels": ["backup"],
}
},
"modelPresets": {
"primary": {"model": "primary-model", "provider": "primary_edge"},
"backup": {"model": "backup-model", "provider": "backup_edge"},
},
"providers": {
"primary_edge": {
"apiKey": "primary-key",
"apiBase": "https://primary.example/v1",
},
"backup_edge": {
"apiKey": "backup-key",
"apiBase": "https://backup.example/v1",
},
},
})
snapshot = build_provider_snapshot(config)
assert isinstance(snapshot.provider, FallbackProvider)
assert snapshot.provider._primary.provider_name == "primary_edge"
fallback = snapshot.provider._provider_factory(snapshot.provider._fallback_presets[0])
assert fallback.provider_name == "backup_edge"
def test_inline_fallback_reasoning_effort_does_not_inherit_primary() -> None:
from nanobot.config.schema import Config, InlineFallbackConfig
from nanobot.providers.factory import provider_signature
config = Config.model_validate({
"agents": {
"defaults": {
"modelPreset": "fast",
"fallbackModels": [
{"provider": "openai", "model": "gpt-4.1"}
],
}
},
"modelPresets": {
"fast": {
"model": "anthropic/claude-opus-4-5",
"provider": "anthropic",
"reasoningEffort": "high",
}
},
"providers": {
"anthropic": {"apiKey": "primary-key"},
"openai": {"apiKey": "fallback-key"},
},
})
signature = provider_signature(config)
fallback_signatures = signature[-1]
config.model_presets["fast"].reasoning_effort = "low"
assert provider_signature(config)[-1] == fallback_signatures
config.agents.defaults.fallback_models = [
InlineFallbackConfig(provider="openai", model="gpt-4.1", reasoning_effort="low")
]
assert provider_signature(config)[-1] != fallback_signatures
class TestNoFallbackWhenPrimarySucceeds:
@pytest.mark.asyncio
async def test(self) -> None:
primary = _FakeProvider("primary", _make_response("primary ok"))
factory = MagicMock()
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "primary ok"
assert result.finish_reason == "stop"
factory.assert_not_called()
class _RaisingProvider(LLMProvider):
"""Provider whose chat/chat_stream raise, like an auth/setup failure."""
def __init__(self, name: str = "raiser", exc: BaseException | None = None):
super().__init__(provider_name=name)
self.name = name
self._exc = exc if exc is not None else RuntimeError("GitHub Copilot is not logged in.")
def get_default_model(self) -> str:
return f"{self.name}/model"
async def chat(self, **kwargs: Any) -> LLMResponse:
raise self._exc
async def chat_stream(self, **kwargs: Any) -> LLMResponse:
raise self._exc
class _StreamingThenRaisingProvider(_RaisingProvider):
async def chat_stream(self, **kwargs: Any) -> LLMResponse:
on_content_delta = kwargs.get("on_content_delta")
if on_content_delta:
await on_content_delta("partial")
raise self._exc
class TestFallbackWhenPrimaryRaises:
@pytest.mark.parametrize(
"exc",
[TimeoutError(), httpx.ReadTimeout(""), httpx.ConnectError(""), httpx.ReadError("")],
ids=["asyncio-timeout", "httpx-timeout", "connection", "httpx-network-error"],
)
@pytest.mark.asyncio
async def test_transient_exception_triggers_fallback(self, exc: Exception) -> None:
primary = _RaisingProvider("primary", exc)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}], model="primary-model")
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
factory.assert_called_once_with(_fallback("fallback-a"))
@pytest.mark.asyncio
async def test_primary_exception_triggers_fallback(self) -> None:
"""A primary whose chat() raises must not abort failover.
GitHubCopilotProvider.chat refreshes its token before the request and
raises when not logged in; the exception must be treated as a
fallbackable primary error, not swallow the whole fallback chain.
"""
primary = _RaisingProvider("primary")
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}], model="primary-model")
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
factory.assert_called_once_with(_fallback("fallback-a"))
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True], ids=["chat", "stream"])
async def test_retry_entry_point_preserves_transient_exception_metadata(
self,
stream: bool,
) -> None:
primary = _RaisingProvider("primary", TimeoutError())
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
retry = fb.chat_stream_with_retry if stream else fb.chat_with_retry
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock) as sleep:
result = await retry(messages=[{"role": "user", "content": "hi"}], model="primary-model")
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
factory.assert_called_once_with(_fallback("fallback-a"))
assert sleep.await_count == 3
assert all(
ceiling / 2 <= request.args[0] <= ceiling
for request, ceiling in zip(sleep.await_args_list, (1, 2, 4))
)
@pytest.mark.asyncio
async def test_authentication_exception_message_is_classified(self) -> None:
primary = _RaisingProvider("primary")
response, exception = await FallbackProvider._call_provider(
lambda provider, kwargs: provider.chat(**kwargs),
primary,
{},
)
assert exception is primary._exc
assert response.error_kind == "authentication"
@pytest.mark.asyncio
async def test_non_string_exception_metadata_is_normalized_before_fallback(self) -> None:
class NumericMetadataError(Exception):
error_type = 429
error_code = 429
status_code = 429
primary = _RaisingProvider("primary", NumericMetadataError("rate limited"))
response, exception = await FallbackProvider._call_provider(
lambda provider, kwargs: provider.chat(**kwargs),
primary,
{},
)
assert exception is primary._exc
assert response.error_type == "429"
assert response.error_code == "429"
assert FallbackProvider._should_fallback(response) is True
@pytest.mark.parametrize(
"exc",
[
ValueError("unexpected response shape"),
PermissionError("local provider file access failed"),
],
ids=["value-error", "local-permission-error"],
)
@pytest.mark.asyncio
async def test_non_fallbackable_primary_exception_does_not_trigger_fallback(
self, exc: Exception,
) -> None:
primary = _RaisingProvider("primary", exc)
factory = MagicMock(
return_value=_FakeProvider("fallback", _make_response("fallback"))
)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(
messages=[{"role": "user", "content": "hi"}],
model="primary-model",
)
assert result.finish_reason == "error"
assert result.content is not None
assert str(exc) in result.content
factory.assert_not_called()
@pytest.mark.asyncio
async def test_fallback_exception_advances_to_next_fallback(self) -> None:
primary = _FakeProvider("primary", _error_response())
raising_fb = _RaisingProvider("fb1")
ok_fb = _FakeProvider("fb2", _make_response("second fallback ok"))
factory = MagicMock(side_effect=[raising_fb, ok_fb])
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a"), _fallback("fallback-b")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}], model="primary-model")
assert result.content == "second fallback ok"
assert result.finish_reason == "stop"
@pytest.mark.asyncio
async def test_primary_exception_stream_triggers_fallback(self) -> None:
primary = _RaisingProvider("primary")
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat_stream(messages=[{"role": "user", "content": "hi"}], model="primary-model")
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
@pytest.mark.asyncio
async def test_primary_exception_after_streaming_does_not_duplicate_output(self) -> None:
primary = _StreamingThenRaisingProvider("primary")
factory = MagicMock(return_value=_FakeProvider("fallback", _make_response("duplicate")))
deltas: list[str] = []
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
async def collect_delta(text: str) -> None:
deltas.append(text)
result = await fb.chat_stream(
messages=[{"role": "user", "content": "hi"}],
model="primary-model",
on_content_delta=collect_delta,
)
assert deltas == ["partial"]
assert result.finish_reason == "error"
factory.assert_not_called()
@pytest.mark.asyncio
async def test_primary_cancellation_is_not_converted_to_failover(self) -> None:
primary = _RaisingProvider("primary", asyncio.CancelledError())
factory = MagicMock(return_value=_FakeProvider("fallback", _make_response("fallback")))
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
with pytest.raises(asyncio.CancelledError):
await fb.chat(messages=[{"role": "user", "content": "hi"}])
factory.assert_not_called()
class TestFallbackOnPrimaryError:
@pytest.mark.asyncio
async def test_first_fallback_succeeds(self) -> None:
primary = _FakeProvider("primary", _error_response())
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}], model="primary-model")
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
factory.assert_called_once_with(_fallback("fallback-a"))
assert primary.chat_calls[0]["model"] == "primary-model"
assert fallback.chat_calls[0]["model"] == "fallback-a"
@pytest.mark.asyncio
async def test_primary_compaction_uses_primary_context_window(self) -> None:
primary = _FakeProvider("primary", _make_response("primary ok"))
primary.compact = True
fb = FallbackProvider(
primary=primary,
fallback_presets=[
_fallback("small-chat", context_window_tokens=50_000),
],
provider_factory=MagicMock(),
primary_context_window_tokens=200_000,
)
await fb.chat_with_context(
messages=[{"role": "user", "content": "hi"}],
model="gpt-5.6",
max_tokens=10_000,
provider_context=ProviderCallContext(
context_window_tokens=50_000,
session_id="webui:cache-test",
),
)
primary_context = primary.context_calls[0]
assert primary_context is not None
assert primary_context.context_window_tokens == 200_000
assert primary_context.session_id == "webui:cache-test"
assert resolve_compact_threshold(
primary_context.context_window_tokens,
10_000,
) == 180_000
@pytest.mark.asyncio
async def test_native_fallback_compaction_uses_its_own_context_window(self) -> None:
primary = _FakeProvider("primary", _error_response())
primary.compact = True
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
fallback.compact = True
fb = FallbackProvider(
primary=primary,
fallback_presets=[
_fallback("fallback-a", context_window_tokens=120_000),
],
provider_factory=MagicMock(return_value=fallback),
primary_context_window_tokens=200_000,
)
result = await fb.chat_with_context(
messages=[{"role": "user", "content": "hi"}],
model="gpt-5.6",
provider_context=ProviderCallContext(context_window_tokens=50_000),
)
assert result.content == "fallback ok"
assert primary.context_calls == [
ProviderCallContext(context_window_tokens=200_000)
]
assert fallback.context_calls == [
ProviderCallContext(context_window_tokens=120_000)
]
@pytest.mark.asyncio
async def test_native_fallback_gets_context_when_primary_does_not_use_it(self) -> None:
primary = _FakeProvider("primary", _error_response())
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
fallback.compact = True
fb = FallbackProvider(
primary=primary,
fallback_presets=[
_fallback("fallback-a", context_window_tokens=120_000),
],
provider_factory=MagicMock(return_value=fallback),
primary_context_window_tokens=200_000,
)
messages = [{"role": "user", "content": "hi"}]
controller = ProviderConversationStateController(
provider=fb,
model="primary-model",
messages=messages,
)
assert fb.supports_native_compaction("primary-model") is False
provider_context = controller.prepare_request(
messages,
context_window_tokens=50_000,
)
assert provider_context == ProviderCallContext(
context_window_tokens=50_000
)
result = await fb.chat_with_context(
messages=messages,
model="primary-model",
provider_context=provider_context,
)
assert result.content == "fallback ok"
assert primary.context_calls == [ProviderCallContext()]
assert fallback.context_calls == [
ProviderCallContext(context_window_tokens=120_000)
]
@pytest.mark.asyncio
async def test_responses_chat_fallback_responses_rebuilds_state(self) -> None:
primary = _FakeProvider("primary", _error_response())
primary.resumable = True
primary.compact = True
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
messages = [{"role": "user", "content": "hi"}]
state = ProviderConversationState(
kind="openai_responses",
provider="openai:test",
model="gpt-5.6",
version=1,
payload={"items": [{"type": "reasoning", "encrypted_content": "opaque"}]},
pending_messages=list(messages),
)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=MagicMock(return_value=fallback),
)
controller = ProviderConversationStateController(
provider=fb,
model="gpt-5.6",
messages=messages,
state=state,
)
provider_context = controller.prepare_request(
messages,
context_window_tokens=200_000,
)
assert provider_context is not None
result = await fb.chat_with_context(
messages=messages,
model="gpt-5.6",
provider_context=provider_context,
)
assert result.content == "fallback ok"
assert primary.context_calls == [provider_context]
assert fallback.context_calls == [ProviderCallContext()]
assert fallback.chat_calls[0]["messages"] == messages
controller.observe_response(result, messages)
messages.append({"role": "assistant", "content": result.content})
assert controller.finish(messages) is None
recovered_state = ProviderConversationState(
kind="openai_responses",
provider="openai:test",
model="gpt-5.6",
version=1,
payload={"items": [{"type": "reasoning", "encrypted_content": "recovered"}]},
)
primary._response = LLMResponse(
content="primary recovered",
provider_state=recovered_state,
)
next_turn = ProviderConversationStateController(
provider=fb,
model="gpt-5.6",
messages=messages,
)
next_context = next_turn.prepare_request(
messages,
context_window_tokens=200_000,
)
assert next_context == ProviderCallContext(context_window_tokens=200_000)
recovered = await fb.chat_with_context(
messages=messages,
model="gpt-5.6",
provider_context=next_context,
)
assert recovered.provider_state is recovered_state
assert primary.context_calls[-1] == next_context
assert primary.chat_calls[-1]["messages"] == messages
@pytest.mark.asyncio
@pytest.mark.parametrize(
("primary_error_kind", "primary_status", "primary_should_retry"),
[
("server_error", 503, True),
("authentication", 401, False),
],
ids=["transient", "authentication"],
)
async def test_final_fallback_error_uses_primary_state_disposition(
self,
primary_error_kind: str,
primary_status: int,
primary_should_retry: bool,
) -> None:
primary = _FakeProvider(
"primary",
_make_response(
"primary unavailable",
finish_reason="error",
error_kind=primary_error_kind,
error_status_code=primary_status,
error_should_retry=primary_should_retry,
),
)
primary.resumable = True
fallback = _FakeProvider(
"fallback",
_make_response(
"fallback invalid request",
finish_reason="error",
error_kind="invalid_request",
error_status_code=400,
error_should_retry=False,
),
)
messages = [{"role": "user", "content": "continue"}]
state = ProviderConversationState(
kind="openai_responses",
provider="openai:test",
model="gpt-5.6",
version=1,
payload={"items": [{"type": "reasoning", "encrypted_content": "opaque"}]},
pending_messages=list(messages),
)
provider = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=MagicMock(return_value=fallback),
)
controller = ProviderConversationStateController(
provider=provider,
model="gpt-5.6",
messages=messages,
state=state,
)
provider_context = controller.prepare_request(
messages,
context_window_tokens=200_000,
)
assert provider_context is not None
response = await provider.chat_with_context(
messages=messages,
model="gpt-5.6",
provider_context=provider_context,
)
controller.observe_response(response, messages)
assert response.content == "fallback invalid request"
assert response.preserve_provider_state_on_error is True
restored = controller.finish(messages)
assert restored is not None
assert restored.payload == state.payload
@pytest.mark.asyncio
async def test_reports_only_the_successful_fallback_model(self) -> None:
primary = _FakeProvider("primary", _error_response())
failed_fallback = _FakeProvider("failed", _error_response("backup overloaded"))
successful_fallback = _FakeProvider("fallback", _make_response("fallback ok"))
fallback_models: list[str] = []
async def _observe(selection: FallbackModelSelection) -> None:
fallback_models.append(selection.model)
assert selection.reauth_provider is None
fb = FallbackProvider(
primary=primary,
fallback_presets=[
_fallback("fallback-a", provider="backup"),
_fallback("fallback-b", provider="backup"),
],
provider_factory=MagicMock(side_effect=[failed_fallback, successful_fallback]),
fallback_model_observer=_observe,
)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock) as sleep:
result = await fb.chat_with_retry(
messages=[{"role": "user", "content": "hi"}],
model="primary-model",
)
assert result.content == "fallback ok"
assert fallback_models == ["fallback-b"]
assert sleep.await_count == 3
assert all(
ceiling / 2 <= request.args[0] <= ceiling
for request, ceiling in zip(sleep.await_args_list, (1, 2, 4))
)
@pytest.mark.asyncio
@pytest.mark.parametrize(("provider_name", "status", "code", "kind", "expected"), [
("openai_codex", 401, "token_revoked", "http", "openai_codex"),
("xai_grok", 401, None, "http", "xai_grok"),
("github_copilot", 401, None, "http", "github_copilot"),
("openai_codex", 400, "invalid_grant", "http", "openai_codex"),
("openai_codex", 403, "token_revoked", "http", "openai_codex"),
("openai_codex", None, None, "oauth_auth_required", "openai_codex"),
("openai_codex", 403, "insufficient_scope", "permission", None),
("openai_codex", 429, None, "rate_limit", None),
("openai_codex", 503, None, "server_error", None),
("openai_codex", None, None, "timeout", None),
("openai_codex", None, None, "authentication", None),
("openai", 401, "invalid_api_key", "authentication", None),
])
async def test_reports_only_confirmed_oauth_rejections(
self, provider_name, status, code, kind, expected,
) -> None:
observer = AsyncMock()
primary = _FakeProvider(provider_name, _make_response(
"synthetic-secret invalid_token", "error", error_status_code=status,
error_code=code, error_kind=kind,
))
provider = FallbackProvider(
primary=primary, fallback_presets=[_fallback("backup")],
provider_factory=MagicMock(return_value=_FakeProvider("backup")),
fallback_model_observer=observer,
)
await provider.chat(messages=[{"role": "user", "content": "hi"}])
observer.assert_awaited_once_with(FallbackModelSelection("backup", expected))
assert "synthetic-secret" not in repr(observer.await_args)
@pytest.mark.asyncio
async def test_does_not_reuse_auth_diagnosis_when_circuit_skips_primary(self) -> None:
observer = AsyncMock()
provider = FallbackProvider(
primary=_FakeProvider("openai_codex", _make_response("revoked", "error", error_status_code=401)),
fallback_presets=[_fallback("backup")],
provider_factory=MagicMock(return_value=_FakeProvider("backup")),
fallback_model_observer=observer,
)
for _ in range(4):
await provider.chat(messages=[{"role": "user", "content": "hi"}])
assert [call.args[0].reauth_provider for call in observer.await_args_list] == [
"openai_codex", "openai_codex", "openai_codex", None,
]
@pytest.mark.asyncio
async def test_no_reauth_notice_when_all_fallbacks_fail(self) -> None:
observer = AsyncMock()
provider = FallbackProvider(
primary=_FakeProvider("openai_codex", _make_response("revoked", "error", error_status_code=401)),
fallback_presets=[_fallback("backup")],
provider_factory=MagicMock(return_value=_FakeProvider("backup", _error_response())),
fallback_model_observer=observer,
)
result = await provider.chat(messages=[{"role": "user", "content": "hi"}])
assert result.finish_reason == "error"
observer.assert_not_awaited()
@pytest.mark.asyncio
async def test_logs_primary_error_before_fallback(self) -> None:
primary = _FakeProvider("primary", _error_response("primary overloaded"))
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
logs: list[str] = []
sink_id = logger.add(lambda message: logs.append(str(message)), format="{message}")
try:
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
await fb.chat(messages=[{"role": "user", "content": "hi"}], model="primary-model")
finally:
logger.remove(sink_id)
assert any(
"Primary model 'primary-model' failed: primary overloaded; trying fallback 'fallback-a'"
in line
for line in logs
)
class TestNoFallbackWhenContentStreamed:
@pytest.mark.asyncio
async def test_non_timeout_error_skips_failover(self) -> None:
primary = _FakeProvider("primary", _error_response())
factory = MagicMock()
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
async def _delta(text: str) -> None:
pass
result = await fb.chat_stream(
messages=[{"role": "user", "content": "hi"}],
on_content_delta=_delta,
)
assert result.finish_reason == "error"
factory.assert_not_called()
class TestFallbackOnStreamStalledAfterContent:
@pytest.mark.asyncio
async def test_timeout_with_streamed_content_falls_back(self) -> None:
primary = _FakeProvider(
"primary",
_make_response("stream stalled", finish_reason="error", error_kind="timeout"),
)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
streamed: list[str] = []
recoveries: list[str] = []
async def _delta(text: str) -> None:
streamed.append(text)
async def _recover() -> None:
recoveries.append("recover")
result = await fb.chat_stream(
messages=[{"role": "user", "content": "hi"}],
on_content_delta=_delta,
on_stream_recover=_recover,
)
assert result.finish_reason == "stop"
assert result.content == "fallback ok"
factory.assert_called_once_with(_fallback("fallback-a"))
assert streamed == ["stream stalled", "fallback ok"]
assert recoveries == ["recover"]
class TestFailoverOnEmptyChoices:
@pytest.mark.asyncio
async def test_empty_choices_text_fallback(self) -> None:
from nanobot.providers.fallback_provider import FallbackProvider
response = _make_response(
"Error: API returned empty choices.",
finish_reason="error",
error_kind="empty",
)
assert FallbackProvider._should_fallback(response)
@pytest.mark.asyncio
async def test_empty_choices_no_error_kind_text_fallback(self) -> None:
from nanobot.providers.fallback_provider import FallbackProvider
response = _make_response(
"Error: API returned empty choices.",
finish_reason="error",
)
assert FallbackProvider._should_fallback(response)
@pytest.mark.asyncio
async def test_empty_choices_triggers_failover(self) -> None:
primary = _FakeProvider(
"primary",
_make_response("Error: API returned empty choices.", finish_reason="error"),
)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
factory.assert_called_once()
class TestFailoverOnTransientError:
@pytest.mark.asyncio
async def test_rate_limit(self) -> None:
primary = _FakeProvider("primary", _error_response("rate limit exceeded"))
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
factory.assert_called_once_with(_fallback("fallback-a"))
class TestRetryBeforeFailover:
@pytest.mark.parametrize("stream", [False, True])
async def test_scoped_retry_events_preserve_fallback_chain_ownership(self, stream):
from nanobot.events import EventSink, RetryWaitEvent
primary = _FakeProvider("primary", _retryable_error("primary unavailable"))
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
provider = FallbackProvider(primary, [_fallback("fallback-a")],
MagicMock(return_value=fallback))
observe = AsyncMock()
call = provider.chat_stream_with_retry if stream else provider.chat_with_retry
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await call(
[{"role": "user", "content": "hi"}],
provider_context=ProviderCallContext(events=EventSink(observe)),
)
assert result.content == "fallback ok"
events = [call.args[0] for call in observe.await_args_list]
notices = [event for event in events if isinstance(event, RetryWaitEvent)]
assert len(notices) == 3
assert not any("giving up" in event.content for event in notices)
statuses = [event.state for event in events if isinstance(event, RetryStatusEvent)]
assert statuses == ["cleared", "waiting", "waiting", "waiting", "cleared"]
async def test_scoped_persistent_retry_includes_chain_wait_and_one_terminal(self):
from nanobot.events import EventSink, RetryWaitEvent
primary = _FakeProvider("primary", _retryable_error("primary unavailable"))
fallback = _FakeProvider("fallback", _retryable_error("fallback unavailable"))
provider = FallbackProvider(primary, [_fallback("fallback-a")],
MagicMock(return_value=fallback))
provider._PERSISTENT_IDENTICAL_ERROR_LIMIT = 2
observe = AsyncMock()
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_with_retry(
[{"role": "user", "content": "hi"}], retry_mode="persistent",
provider_context=ProviderCallContext(events=EventSink(observe)),
)
assert result.finish_reason == "error"
events = [call.args[0] for call in observe.await_args_list]
notices = [event.content for event in events if isinstance(event, RetryWaitEvent)]
assert len(notices) == 14
assert sum("Persistent retry stopped" in text for text in notices) == 1
assert not any("giving up" in text for text in notices)
statuses = [event for event in events if isinstance(event, RetryStatusEvent)]
assert sum(event.state == "exhausted" for event in statuses) == 1
assert statuses[-1].state == "exhausted"
@pytest.mark.asyncio
@pytest.mark.parametrize("retry_mode", ["standard", "persistent"])
async def test_primary_recovers_before_fallback(self, retry_mode: str) -> None:
primary = _FakeProvider(
"primary",
responses=[_error_response("rate limited"), _make_response("primary ok")],
)
factory = MagicMock()
provider = FallbackProvider(primary, [_fallback("fallback-a")], factory)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_with_retry(
[{"role": "user", "content": "hi"}],
retry_mode=retry_mode,
)
assert result.content == "primary ok"
assert len(primary.chat_calls) == 2
factory.assert_not_called()
@pytest.mark.asyncio
async def test_primary_exhausts_before_fallback_without_terminal_event(self) -> None:
primary = _FakeProvider(
"primary",
responses=[_retryable_error(f"attempt {attempt}") for attempt in range(4)],
)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
retry_events = AsyncMock()
retry_statuses: list[RetryStatusEvent] = []
provider = FallbackProvider(primary, [_fallback("fallback-a")], factory)
async def _record_status(status: RetryStatusEvent) -> None:
retry_statuses.append(status)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_with_retry(
[{"role": "user", "content": "hi"}],
on_retry_wait=retry_events,
on_retry_status=_record_status,
)
assert result.content == "fallback ok"
assert len(primary.chat_calls) == 4
assert not any("giving up" in call.args[0] for call in retry_events.await_args_list)
assert [status.state for status in retry_statuses] == [
"waiting",
"waiting",
"waiting",
"cleared",
]
assert retry_statuses[-1] == RetryStatusEvent(
state="cleared",
attempt=4,
max_attempts=4,
error_kind="server",
)
factory.assert_called_once_with(_fallback("fallback-a"))
@pytest.mark.asyncio
async def test_all_candidates_exhaust_emit_one_terminal_event(self) -> None:
primary = _FakeProvider("primary", _retryable_error("primary unavailable"))
fallback = _FakeProvider("fallback", _retryable_error("fallback unavailable"))
retry_events = AsyncMock()
terminal_event = AsyncMock()
retry_statuses: list[RetryStatusEvent] = []
provider = FallbackProvider(
primary,
[_fallback("fallback-a")],
MagicMock(return_value=fallback),
)
async def _record_status(status: RetryStatusEvent) -> None:
retry_statuses.append(status)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_with_retry(
[{"role": "user", "content": "hi"}],
on_retry_wait=retry_events,
on_retry_exhausted=terminal_event,
on_retry_status=_record_status,
)
assert result.finish_reason == "error"
assert len(primary.chat_calls) == len(fallback.chat_calls) == 4
assert not any("giving up" in call.args[0] for call in retry_events.await_args_list)
terminal_event.assert_awaited_once_with(
"Model request failed after 4 attempts, giving up."
)
assert "cleared" in [status.state for status in retry_statuses]
assert retry_statuses[-1].state == "exhausted"
@pytest.mark.asyncio
async def test_fallback_auth_error_preserves_primary_diagnosis_and_exhaustion(self) -> None:
primary = _FakeProvider("primary", _make_response(
"private network detail", finish_reason="error", error_kind="connection",
))
fallback = _FakeProvider("fallback", _make_response(
"private auth detail", finish_reason="error", error_status_code=401,
))
provider = FallbackProvider(primary, [_fallback("backup")], MagicMock(return_value=fallback))
statuses: list[RetryStatusEvent] = []
async def record(status: RetryStatusEvent) -> None:
statuses.append(status)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_with_retry(
[{"role": "user", "content": "hello"}], on_retry_status=record,
)
assert len(primary.chat_calls) == 4
assert len(fallback.chat_calls) == 1
assert LLMProvider.public_error_kind(result) == "connection"
assert result.preserve_provider_state_on_error is True
assert not LLMProvider.is_transient_response(result)
assert statuses[-1] == RetryStatusEvent(
state="exhausted", attempt=4, max_attempts=4, error_kind="connection",
)
@pytest.mark.asyncio
@pytest.mark.parametrize("factory_fails", [False, True])
async def test_persistent_mode_repeats_the_whole_chain(
self,
factory_fails: bool,
) -> None:
primary = _FakeProvider("primary", _retryable_error("primary unavailable"))
fallback = _FakeProvider("fallback", _retryable_error("fallback unavailable"))
factory = (
MagicMock(side_effect=ValueError("missing fallback credentials"))
if factory_fails
else MagicMock(return_value=fallback)
)
terminal_event = AsyncMock()
provider = FallbackProvider(primary, [_fallback("fallback-a")], factory)
provider._PERSISTENT_IDENTICAL_ERROR_LIMIT = 2
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_with_retry(
[{"role": "user", "content": "hi"}],
retry_mode="persistent",
on_retry_exhausted=terminal_event,
)
assert result.finish_reason == "error"
assert len(primary.chat_calls) == 8
assert len(fallback.chat_calls) == (0 if factory_fails else 8)
assert factory.call_count == 2
terminal_event.assert_awaited_once_with(
"Persistent retry stopped after 2 identical errors."
)
@pytest.mark.asyncio
async def test_open_primary_circuit_remains_retryable_when_factory_fails(self) -> None:
primary = _FakeProvider("primary")
factory = MagicMock(side_effect=ValueError("missing fallback credentials"))
terminal_event = AsyncMock()
provider = FallbackProvider(primary, [_fallback("fallback-a")], factory)
provider._primary_tripped_at = 100.0
provider._PERSISTENT_IDENTICAL_ERROR_LIMIT = 2
with (
patch("nanobot.providers.fallback_provider.time.monotonic", return_value=100.0),
patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock),
):
result = await provider.chat_with_retry(
[{"role": "user", "content": "hi"}],
retry_mode="persistent",
on_retry_exhausted=terminal_event,
)
assert result.error_should_retry is True
assert result.error_retry_after_s == 60
assert primary.chat_calls == []
assert factory.call_count == 2
terminal_event.assert_awaited_once_with(
"Persistent retry stopped after 2 identical errors."
)
@pytest.mark.asyncio
async def test_fallback_retries_before_trying_next_model(self) -> None:
primary = _FakeProvider(
"primary",
_make_response(
"unauthorized",
finish_reason="error",
error_kind="authentication",
error_should_retry=False,
),
)
fallback_a = _FakeProvider(
"fallback-a",
responses=[_error_response("rate limited"), _make_response("fallback a ok")],
)
factory = MagicMock(side_effect=[fallback_a, _FakeProvider("fallback-b")])
fallback_a_preset = _fallback("fallback-a")
provider = FallbackProvider(
primary,
[fallback_a_preset, _fallback("fallback-b")],
factory,
)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_with_retry([{"role": "user", "content": "hi"}])
assert result.content == "fallback a ok"
assert len(fallback_a.chat_calls) == 2
factory.assert_called_once_with(fallback_a_preset)
@pytest.mark.asyncio
async def test_stream_recovery_keeps_fallback_eligible(self) -> None:
primary = _FakeProvider(
"primary",
responses=[
_make_response("partial", finish_reason="error", error_kind="timeout"),
*[_retryable_error() for _ in range(3)],
],
)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
streamed = AsyncMock()
recovered = AsyncMock()
provider = FallbackProvider(primary, [_fallback("fallback-a")], factory)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_stream_with_retry(
[{"role": "user", "content": "hi"}],
on_content_delta=streamed,
on_stream_recover=recovered,
)
assert result.content == "fallback ok"
assert len(primary.chat_stream_calls) == 4
assert [call.args[0] for call in streamed.await_args_list] == ["partial", "fallback ok"]
recovered.assert_awaited_once_with()
factory.assert_called_once_with(_fallback("fallback-a"))
@pytest.mark.asyncio
async def test_stream_without_delta_callback_retries_before_fallback(self) -> None:
primary = _FakeProvider(
"primary",
responses=[_retryable_error(f"attempt {attempt}") for attempt in range(4)],
)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
provider = FallbackProvider(primary, [_fallback("fallback-a")], factory)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_stream_with_retry(
[{"role": "user", "content": "hi"}]
)
assert result.content == "fallback ok"
assert len(primary.chat_stream_calls) == 4
factory.assert_called_once_with(_fallback("fallback-a"))
@pytest.mark.asyncio
async def test_unrecovered_stream_keeps_non_timeout_fallback_blocked(self) -> None:
primary = _FakeProvider(
"primary",
responses=[
_make_response("partial", finish_reason="error", error_kind="timeout"),
_retryable_error(),
_retryable_error(),
_retryable_error("last error"),
],
)
factory = MagicMock()
streamed = AsyncMock()
provider = FallbackProvider(primary, [_fallback("fallback-a")], factory)
with patch("nanobot.providers.base.asyncio.sleep", new_callable=AsyncMock):
result = await provider.chat_stream_with_retry(
[{"role": "user", "content": "hi"}],
on_content_delta=streamed,
)
assert result.content == "last error"
streamed.assert_awaited_once_with("partial")
factory.assert_not_called()
class TestFailoverOnArrearageError:
@pytest.mark.asyncio
async def test_non_retryable_quota_tries_configured_fallback(self) -> None:
arrearage = _make_response(
"insufficient quota",
finish_reason="error",
error_status_code=429,
error_type="insufficient_quota",
error_should_retry=False,
)
primary = _FakeProvider("primary", arrearage)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
fallback_preset = _fallback("fallback-a")
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[fallback_preset],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
factory.assert_called_once_with(fallback_preset)
@pytest.mark.asyncio
async def test_insufficient_credits_on_400_tries_configured_fallback(self) -> None:
arrearage = _make_response(
"[400]: You have insufficient credits to make this request.",
finish_reason="error",
error_status_code=400,
error_code="BAD_REQUEST",
error_type="invalid_request_error",
error_should_retry=False,
)
primary = _FakeProvider("primary", arrearage)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
fallback_preset = _fallback("fallback-a")
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[fallback_preset],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
factory.assert_called_once_with(fallback_preset)
@pytest.mark.asyncio
async def test_plain_bad_request_still_does_not_fall_back(self) -> None:
bad_request = _make_response(
"invalid parameter: temperature must be <= 2",
finish_reason="error",
error_status_code=400,
error_code="BAD_REQUEST",
error_type="invalid_request_error",
error_should_retry=False,
)
primary = _FakeProvider("primary", bad_request)
factory = MagicMock()
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result is bad_request
factory.assert_not_called()
@pytest.mark.asyncio
async def test_without_fallback_presets_returns_original_error(self) -> None:
arrearage = _make_response(
"payment required",
finish_reason="error",
error_status_code=402,
error_should_retry=False,
)
primary = _FakeProvider("primary", arrearage)
factory = MagicMock()
fb = FallbackProvider(
primary=primary,
fallback_presets=[],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result is arrearage
factory.assert_not_called()
class TestFailoverOnAuthenticationError:
@pytest.mark.parametrize(
"authentication_error",
[
pytest.param(
_make_response(
(
"Error: {'error': {'type': 'authentication_error', "
"'message': 'The API Key appears to be invalid or may have expired.'}}"
),
finish_reason="error",
error_type="authentication_error",
error_should_retry=False,
),
id="authentication-error-type",
),
pytest.param(
_make_response(
"unauthorized",
finish_reason="error",
error_status_code=401,
error_kind="http",
error_should_retry=False,
),
id="http-401",
),
pytest.param(
_make_response(
"bad key",
finish_reason="error",
error_type="invalid_request_error",
error_code="invalid_api_key",
error_should_retry=False,
),
id="invalid-api-key-code",
),
pytest.param(
_make_response(
"credentials have expired",
finish_reason="error",
error_should_retry=False,
),
id="expired-credentials-text",
),
pytest.param(
_make_response(
"permission denied",
finish_reason="error",
error_status_code=403,
error_kind="permission",
error_should_retry=False,
),
id="permission-error",
),
],
)
@pytest.mark.asyncio
async def test_tries_configured_fallback(
self,
authentication_error: LLMResponse,
) -> None:
primary = _FakeProvider("primary", authentication_error)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
fallback_preset = _fallback("fallback-a")
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[fallback_preset],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
factory.assert_called_once_with(fallback_preset)
class TestFallbackErrorClassification:
@pytest.mark.asyncio
async def test_bad_request(self) -> None:
primary = _FakeProvider(
"primary",
_make_response(
"invalid request",
finish_reason="error",
error_status_code=400,
error_kind="invalid_request",
),
)
factory = MagicMock()
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.finish_reason == "error"
factory.assert_not_called()
@pytest.mark.asyncio
async def test_content_filter_takes_precedence_over_403(self) -> None:
primary = _FakeProvider(
"primary",
_make_response(
"request blocked by content filter",
finish_reason="error",
error_status_code=403,
error_kind="content_filter",
error_should_retry=False,
),
)
factory = MagicMock()
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.finish_reason == "error"
factory.assert_not_called()
@pytest.mark.asyncio
async def test_timeout_triggers_fallback(self) -> None:
primary = _FakeProvider(
"primary",
_make_response("timed out", finish_reason="error", error_kind="timeout"),
)
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
assert result.finish_reason == "stop"
factory.assert_called_once_with(_fallback("fallback-a"))
class TestFallbackTriesModelsInOrder:
@pytest.mark.asyncio
async def test(self) -> None:
primary = _FakeProvider("primary", _error_response("primary fail"))
fallback_a = _FakeProvider("a", _error_response("a fail"))
fallback_b = _FakeProvider("b", _make_response("b ok"))
factory = MagicMock(side_effect=[fallback_a, fallback_b])
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a"), _fallback("fallback-b")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "b ok"
assert factory.call_count == 2
factory.assert_any_call(_fallback("fallback-a"))
factory.assert_any_call(_fallback("fallback-b"))
class TestAllFallbacksFail:
@pytest.mark.asyncio
async def test(self) -> None:
primary = _FakeProvider("primary", _error_response("primary fail"))
fallback = _FakeProvider("fallback", _error_response("all fail"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.finish_reason == "error"
assert "all fail" in result.content
class TestFactoryExceptionSkipsModel:
@pytest.mark.asyncio
async def test(self) -> None:
primary = _FakeProvider("primary", _error_response())
fallback_b = _FakeProvider("b", _make_response("b ok"))
factory = MagicMock(side_effect=[ValueError("no key"), fallback_b])
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a"), _fallback("fallback-b")],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "b ok"
assert factory.call_count == 2
class TestFallbackModelParameter:
@pytest.mark.asyncio
async def test(self) -> None:
"""Fallback calls should use the fallback model name."""
primary = _FakeProvider("primary", _error_response())
fallback = _FakeProvider("fallback", _make_response("ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-model")],
provider_factory=factory,
)
await fb.chat(messages=[{"role": "user", "content": "hi"}], model="primary-model")
assert fallback.chat_calls[0]["model"] == "fallback-model"
@pytest.mark.asyncio
async def test_uses_fallback_generation_fields(self) -> None:
primary = _FakeProvider("primary", _error_response())
fallback = _FakeProvider("fallback", _make_response("ok"))
fb = FallbackProvider(
primary=primary,
fallback_presets=[
_fallback(
"fallback-model",
max_tokens=1234,
temperature=0.4,
reasoning_effort=None,
)
],
provider_factory=MagicMock(return_value=fallback),
)
await fb.chat(
messages=[{"role": "user", "content": "hi"}],
model="primary-model",
max_tokens=8192,
temperature=0.1,
reasoning_effort="high",
)
assert fallback.chat_calls[0]["model"] == "fallback-model"
assert fallback.chat_calls[0]["max_tokens"] == 1234
assert fallback.chat_calls[0]["temperature"] == 0.4
assert "reasoning_effort" not in fallback.chat_calls[0]
class TestNoFallbackWhenEmptyList:
@pytest.mark.asyncio
async def test(self) -> None:
primary = _FakeProvider("primary", _error_response())
factory = MagicMock()
fb = FallbackProvider(
primary=primary,
fallback_presets=[],
provider_factory=factory,
)
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.finish_reason == "error"
factory.assert_not_called()
@pytest.mark.asyncio
async def test_retry_entrypoints_delegate_to_primary(self) -> None:
primary = _FakeProvider("primary")
provider = FallbackProvider(primary, [], MagicMock())
response = _make_response("primary ok")
with (
patch.object(
primary, "chat_with_retry", new_callable=AsyncMock, return_value=response
) as chat_retry,
patch.object(
primary,
"chat_stream_with_retry",
new_callable=AsyncMock,
return_value=response,
) as stream_retry,
):
assert (await provider.chat_with_retry([])) is response
assert (await provider.chat_stream_with_retry([])) is response
chat_retry.assert_awaited_once()
stream_retry.assert_awaited_once()
class TestChatStreamFailover:
@pytest.mark.asyncio
async def test_fallback_succeeds(self) -> None:
primary = _FakeProvider("primary", _error_response(""))
fallback = _FakeProvider("fallback", _make_response("stream ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
result = await fb.chat_stream(messages=[{"role": "user", "content": "hi"}])
assert result.content == "stream ok"
assert result.finish_reason == "stop"
class TestGetDefaultModel:
def test(self) -> None:
primary = _FakeProvider("primary")
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("a")],
provider_factory=MagicMock(),
)
assert fb.get_default_model() == "primary/model"
async def test_fallback_closes_temporary_provider_and_owns_primary_shutdown() -> None:
primary = _FakeProvider("primary", _error_response())
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
primary.aclose = AsyncMock()
fallback.aclose = AsyncMock()
wrapper = FallbackProvider(primary, [_fallback("backup")], MagicMock(return_value=fallback))
result = await wrapper.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
fallback.aclose.assert_awaited_once()
primary.aclose.assert_not_awaited()
await wrapper.aclose()
primary.aclose.assert_awaited_once()
class TestCircuitBreaker:
@pytest.mark.asyncio
async def test_skips_primary_after_three_failures(self) -> None:
primary = _FakeProvider("primary", _error_response())
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
for _ in range(3):
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
assert len(primary.chat_calls) == 3
primary.chat_calls.clear()
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
assert len(primary.chat_calls) == 0
@pytest.mark.asyncio
async def test_resets_on_success(self) -> None:
primary = _FakeProvider("primary", _error_response())
fallback = _FakeProvider("fallback", _make_response("fallback ok"))
factory = MagicMock(return_value=fallback)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("fallback-a")],
provider_factory=factory,
)
for _ in range(2):
await fb.chat(messages=[{"role": "user", "content": "hi"}])
primary._response = _make_response("primary ok")
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "primary ok"
primary._response = _error_response()
primary.chat_calls.clear()
result = await fb.chat(messages=[{"role": "user", "content": "hi"}])
assert result.content == "fallback ok"
assert len(primary.chat_calls) == 1
class TestGenerationForwarded:
def test(self) -> None:
from nanobot.providers.base import GenerationSettings
primary = _FakeProvider("primary")
primary.generation = GenerationSettings(temperature=0.5, max_tokens=1024)
fb = FallbackProvider(
primary=primary,
fallback_presets=[_fallback("a")],
provider_factory=MagicMock(),
)
assert fb.generation.temperature == 0.5
assert fb.generation.max_tokens == 1024