Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

36.3 KB
Raw Blame
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Tests for VLM failover/backup configuration functionality."""

import time
from unittest.mock import AsyncMock, Mock

import pytest

from openviking.models.vlm.base import (
    FailoverVLM,
    PrimaryBackupSwitcher,
)
from openviking.models.vlm.token_usage import TokenUsageTracker
from openviking_cli.utils.config.vlm_config import VLMConfig


class TestVLMBackupConfig:
    """Tests for VLMConfig backup field validation and migration."""

    def test_backup_config_migrated_to_credentials(self):
        """Test that backup configuration is migrated to credentials list."""
        backup_config = VLMConfig(
            model="backup-model",
            api_key="backup-key",
            provider="openai",
        )
        config = VLMConfig(
            model="primary-model",
            api_key="primary-key",
            provider="volcengine",
            backup=backup_config,
        )
        # After migration, backup should be None but credentials should have 2 entries
        assert config.backup is None
        assert len(config.credentials) == 2
        assert config.credentials[0].id == "legacy-primary"
        assert config.credentials[0].provider == "volcengine"
        assert config.credentials[0].api_key == "primary-key"
        assert config.credentials[0].model == "primary-model"
        assert config.credentials[1].id == "legacy-backup"
        assert config.credentials[1].provider == "openai"
        assert config.credentials[1].api_key == "backup-key"
        assert config.credentials[1].model == "backup-model"

    def test_recursive_backup_config_not_possible(self):
        """Test that recursive backup configurations are automatically prevented by migration.

        With the new multi-credential architecture, when a config with backup is created,
        it gets migrated to credentials format and backup is set to None. This means
        recursive backups are automatically prevented.
        """
        nested_backup = VLMConfig(
            model="nested-model",
            api_key="nested-key",
            provider="openai",
        )
        backup_config = VLMConfig(
            model="backup-model",
            api_key="backup-key",
            provider="openai",
            backup=nested_backup,
        )
        # After migration, backup_config.backup should be None
        assert backup_config.backup is None
        # And it should have 2 credentials (primary + backup)
        assert len(backup_config.credentials) == 2

        # Now creating a config with backup=backup_config works (no recursion)
        # because backup_config.backup is already None
        config = VLMConfig(
            model="primary-model",
            api_key="primary-key",
            provider="volcengine",
            backup=backup_config,
        )
        # The outer config gets its own 2 credentials (primary + backup_config's top-level)
        assert len(config.credentials) == 2
        assert config.backup is None

    def test_backup_without_own_backup_allowed(self):
        """Test that backup config without its own backup is allowed."""
        backup_config = VLMConfig(
            model="backup-model",
            api_key="backup-key",
            provider="openai",
        )
        config = VLMConfig(
            model="primary-model",
            api_key="primary-key",
            provider="volcengine",
            backup=backup_config,
        )
        # Should not raise and should migrate to credentials
        assert len(config.credentials) == 2

    def test_per_credential_model_overrides_parent_model(self):
        """Each credential's `model` field should override parent VLMConfig.model
        when building the per-credential VLM config dict."""
        from openviking_cli.utils.config.vlm_config import VLMCredential

        config = VLMConfig(
            model="parent-model",
            credentials=[
                VLMCredential(
                    id="cred-a",
                    provider="volcengine",
                    model="endpoint-a",
                    api_key="key-a",
                    api_base="https://example.com/a",
                ),
                VLMCredential(
                    id="cred-b",
                    provider="volcengine",
                    api_key="key-b",
                    api_base="https://example.com/b",
                ),
            ],
        )

        dict_a = config._build_vlm_config_dict_for_credential(config.credentials[0])
        dict_b = config._build_vlm_config_dict_for_credential(config.credentials[1])

        assert dict_a["model"] == "endpoint-a"
        # Falls back to parent model when credential.model is not set.
        assert dict_b["model"] == "parent-model"

    def test_per_credential_max_tokens_overrides_parent_value(self):
        from openviking_cli.utils.config.vlm_config import VLMCredential

        config = VLMConfig(
            model="parent-model",
            max_tokens=8192,
            credentials=[
                VLMCredential(
                    id="cred-a",
                    provider="volcengine",
                    api_key="key-a",
                    max_tokens=2048,
                ),
                VLMCredential(
                    id="cred-b",
                    provider="volcengine",
                    api_key="key-b",
                ),
            ],
        )

        dict_a = config._build_vlm_config_dict_for_credential(config.credentials[0])
        dict_b = config._build_vlm_config_dict_for_credential(config.credentials[1])

        assert dict_a["max_tokens"] == 2048
        assert dict_b["max_tokens"] == 8192


class TestLegacyProvidersDictMigration:
    """Tests for migrating legacy ``providers: {name: {...}}`` configs into credentials.

    Regression coverage for: when only ``providers`` was set (without top-level
    ``provider``/``api_key``), ``_normalize_credentials`` previously produced a
    credential with provider=None and api_key=None, which made ``is_available()``
    return False even though the config was valid in the legacy code path.
    """

    def test_providers_only_single_provider_migrates_provider_and_api_key(self):
        """``providers={openai: {api_key: sk}}`` should yield a usable default credential."""
        cfg = VLMConfig(
            model="gpt-4o-mini",
            providers={"openai": {"api_key": "sk-test", "api_base": "https://api.example.com"}},
        )

        assert len(cfg.credentials) == 1
        cred = cfg.credentials[0]
        assert cred.id == "default"
        assert cred.provider == "openai"
        assert cred.api_key == "sk-test"
        assert cred.api_base == "https://api.example.com"
        assert cred.model == "gpt-4o-mini"

        # is_available() must return True - this was the original blocking bug
        assert cfg.is_available() is True

    def test_providers_only_default_provider_picks_correct_one(self):
        """When ``default_provider`` is set, the matching providers entry is used."""
        cfg = VLMConfig(
            model="gpt-4o-mini",
            default_provider="openai",
            providers={
                "openai": {"api_key": "sk-openai"},
                "azure": {"api_key": "sk-azure", "api_base": "https://azure.example.com"},
            },
        )

        assert len(cfg.credentials) == 1
        cred = cfg.credentials[0]
        assert cred.provider == "openai"
        assert cred.api_key == "sk-openai"
        assert cfg.is_available() is True

    def test_top_level_provider_with_providers_dict_merges_correctly(self):
        """``provider=openai`` plus ``providers={openai: {api_key: ...}}`` works."""
        cfg = VLMConfig(
            model="gpt-4o",
            provider="openai",
            providers={"openai": {"api_key": "sk-merged"}},
        )

        assert len(cfg.credentials) == 1
        cred = cfg.credentials[0]
        assert cred.provider == "openai"
        assert cred.api_key == "sk-merged"
        assert cfg.is_available() is True

    def test_legacy_top_level_api_key_still_works(self):
        """Pure top-level legacy config (provider + api_key) keeps working."""
        cfg = VLMConfig(
            model="gpt-4o",
            provider="openai",
            api_key="sk-top-level",
        )

        assert len(cfg.credentials) == 1
        cred = cfg.credentials[0]
        assert cred.provider == "openai"
        assert cred.api_key == "sk-top-level"
        assert cfg.is_available() is True

    def test_providers_only_propagates_extra_fields(self):
        """extra_headers, extra_request_body, and api_version all propagate."""
        cfg = VLMConfig(
            model="gpt-4o",
            providers={
                "openai": {
                    "api_key": "sk",
                    "api_base": "https://api.example.com",
                    "api_version": "2024-01-01",
                    "extra_headers": {"X-Trace": "1"},
                    "extra_request_body": {"foo": "bar"},
                }
            },
        )

        cred = cfg.credentials[0]
        assert cred.provider == "openai"
        assert cred.api_key == "sk"
        assert cred.api_base == "https://api.example.com"
        assert cred.api_version == "2024-01-01"
        assert cred.extra_headers == {"X-Trace": "1"}
        assert cred.extra_request_body == {"foo": "bar"}

    def test_explicit_credentials_take_precedence_over_legacy(self):
        """When ``credentials`` is set, legacy providers dict is not migrated again."""
        cfg = VLMConfig(
            model="gpt-4o",
            credentials=[
                {"id": "explicit", "provider": "openai", "api_key": "sk-explicit"},
            ],
            providers={"openai": {"api_key": "sk-legacy"}},
        )

        assert len(cfg.credentials) == 1
        assert cfg.credentials[0].id == "explicit"
        assert cfg.credentials[0].api_key == "sk-explicit"

    def test_legacy_backup_with_primary_providers_dict_resolves_via_match_provider(self):
        """Primary side: ``providers={openai: {...}}`` + backup must yield a usable legacy-primary.

        Regression: previously, backup migration only read top-level
        ``self.provider/self.api_key`` and produced a credential with
        provider=None / api_key=None when the primary used the providers dict.
        """
        cfg = VLMConfig(
            model="gpt-4o",
            providers={"openai": {"api_key": "sk-primary"}},
            backup=VLMConfig(model="gpt-4o-mini", provider="openai", api_key="sk-backup"),
        )

        assert len(cfg.credentials) == 2
        primary = cfg.credentials[0]
        assert primary.id == "legacy-primary"
        assert primary.provider == "openai"
        assert primary.api_key == "sk-primary"
        assert primary.model == "gpt-4o"

        backup = cfg.credentials[1]
        assert backup.id == "legacy-backup"
        assert backup.provider == "openai"
        assert backup.api_key == "sk-backup"
        assert backup.model == "gpt-4o-mini"

        # is_available() must return True
        assert cfg.is_available() is True

    def test_legacy_backup_with_backup_providers_dict_resolves_via_match_provider(self):
        """Backup side: backup using ``providers={...}`` must produce usable legacy-backup."""
        cfg = VLMConfig(
            model="gpt-4o",
            provider="openai",
            api_key="sk-primary",
            backup=VLMConfig(
                model="gpt-4o-mini",
                providers={"openai": {"api_key": "sk-backup"}},
            ),
        )

        assert len(cfg.credentials) == 2
        primary = cfg.credentials[0]
        assert primary.provider == "openai"
        assert primary.api_key == "sk-primary"

        backup = cfg.credentials[1]
        assert backup.id == "legacy-backup"
        assert backup.provider == "openai"
        assert backup.api_key == "sk-backup"

        assert cfg.is_available() is True

    def test_legacy_backup_with_default_provider_resolves(self):
        """Backup using ``default_provider`` to disambiguate is migrated correctly."""
        cfg = VLMConfig(
            model="gpt-4o",
            providers={"openai": {"api_key": "sk-primary"}},
            backup=VLMConfig(
                model="gpt-4o-mini",
                default_provider="azure",
                providers={
                    "openai": {"api_key": "sk-bk-openai"},
                    "azure": {
                        "api_key": "sk-bk-azure",
                        "api_base": "https://azure.example.com",
                        "api_version": "2024-01-01",
                    },
                },
            ),
        )

        assert len(cfg.credentials) == 2
        backup = cfg.credentials[1]
        assert backup.provider == "azure"
        assert backup.api_key == "sk-bk-azure"
        assert backup.api_base == "https://azure.example.com"
        assert backup.api_version == "2024-01-01"

        assert cfg.is_available() is True

    def test_legacy_backup_propagates_extra_fields_via_match_provider(self):
        """Backup migration via providers dict propagates provider-specific fields."""
        cfg = VLMConfig(
            model="gpt-4o",
            providers={
                "openai": {
                    "api_key": "sk-primary",
                    "extra_headers": {"X-Trace": "p"},
                }
            },
            backup=VLMConfig(
                model="gpt-4o-mini",
                providers={
                    "openai": {
                        "api_key": "sk-backup",
                        "extra_headers": {"X-Trace": "b"},
                        "extra_request_body": {"foo": "bar"},
                    }
                },
            ),
        )

        primary = cfg.credentials[0]
        assert primary.api_key == "sk-primary"
        assert primary.extra_headers == {"X-Trace": "p"}

        backup = cfg.credentials[1]
        assert backup.api_key == "sk-backup"
        assert backup.extra_headers == {"X-Trace": "b"}
        assert backup.extra_request_body == {"foo": "bar"}


class TestFailoverVLM:
    """Tests for FailoverVLM wrapper."""

    def test_initialization(self):
        """Test that FailoverVLM initializes correctly with primary and backup."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"

        failover = FailoverVLM(primary, backup)

        assert failover.primary is primary
        assert failover.backup is backup
        assert failover.is_using_backup is False

    def test_primary_success(self):
        """Test that primary is used when it succeeds."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.return_value = "primary response"

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"

        failover = FailoverVLM(primary, backup)

        result = failover.get_completion(prompt="test")

        assert result == "primary response"
        primary.get_completion.assert_called_once()
        backup.get_completion.assert_not_called()
        assert failover.is_using_backup is False

    def test_primary_fails_non_retryable(self):
        """Test that non-retryable errors don't trigger failover."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.side_effect = ValueError("invalid prompt")

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"

        failover = FailoverVLM(primary, backup)

        with pytest.raises(ValueError, match="invalid prompt"):
            failover.get_completion(prompt="test")

        primary.get_completion.assert_called_once()
        backup.get_completion.assert_not_called()
        assert failover.is_using_backup is False

    def test_primary_fails_rate_limit_does_not_failover(self):
        """Test that rate limit errors do not trigger failover to backup."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.side_effect = Exception("rate limit exceeded (429)")

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.return_value = "backup response"

        failover = FailoverVLM(primary, backup)

        with pytest.raises(Exception, match="rate limit exceeded"):
            failover.get_completion(prompt="test")

        primary.get_completion.assert_called_once()
        backup.get_completion.assert_not_called()
        assert failover.is_using_backup is False

    def test_primary_fails_quota_exceeded_fails_to_backup(self):
        """Test that AccountQuotaExceeded triggers immediate failover."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.side_effect = Exception(
            'API Error: 429 {"error":{"code":"AccountQuotaExceeded",'
            '"message":"You have exceeded the 5-hour usage quota. '
            'It will reset at 2026-05-14 17:18:52 +0800 CST."}}'
        )

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.return_value = "backup response"

        failover = FailoverVLM(primary, backup)

        result = failover.get_completion(prompt="test")

        assert result == "backup response"
        primary.get_completion.assert_called_once()
        backup.get_completion.assert_called_once()
        assert failover.is_using_backup is True

    def test_primary_fails_timeout_does_not_failover(self):
        """Test that timeout errors do not trigger failover to backup."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.side_effect = Exception("request timeout")

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.return_value = "backup response"

        failover = FailoverVLM(primary, backup)

        with pytest.raises(Exception, match="request timeout"):
            failover.get_completion(prompt="test")

        assert failover.is_using_backup is False

    def test_primary_fails_server_error_does_not_failover(self):
        """Test that server errors do not trigger failover to backup."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.side_effect = Exception("server error 503")

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.return_value = "backup response"

        failover = FailoverVLM(primary, backup)

        with pytest.raises(Exception, match="server error 503"):
            failover.get_completion(prompt="test")

        assert failover.is_using_backup is False

    def test_both_fail_raises_last_error(self):
        """Test that if both primary and backup fail, the last error is raised."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.side_effect = Exception(
            'API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}'
        )

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.side_effect = Exception("backup also failed")

        failover = FailoverVLM(primary, backup)

        with pytest.raises(Exception, match="backup also failed"):
            failover.get_completion(prompt="test")

        primary.get_completion.assert_called_once()
        backup.get_completion.assert_called_once()

    def test_stays_on_backup_after_switch(self):
        """Test that once switched to backup, subsequent calls use backup."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.side_effect = Exception(
            'API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}'
        )

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.return_value = "backup response"

        failover = FailoverVLM(primary, backup)

        # First call triggers failover
        result1 = failover.get_completion(prompt="test1")
        assert result1 == "backup response"
        assert failover.is_using_backup is True

        # Second call should use backup directly, not try primary again
        backup.get_completion.return_value = "backup response 2"
        result2 = failover.get_completion(prompt="test2")
        assert result2 == "backup response 2"
        assert primary.get_completion.call_count == 1  # Only called once

    def test_vision_completion_failover(self):
        """Test failover works for vision completion."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_vision_completion.side_effect = Exception(
            'API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}'
        )

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_vision_completion.return_value = "backup vision response"

        failover = FailoverVLM(primary, backup)

        result = failover.get_vision_completion(prompt="describe", images=["test.jpg"])

        assert result == "backup vision response"
        primary.get_vision_completion.assert_called_once()
        backup.get_vision_completion.assert_called_once()

    @pytest.mark.asyncio
    async def test_async_completion_failover(self):
        """Test failover works for async completion."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion_async = AsyncMock(
            side_effect=Exception('API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}')
        )

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion_async = AsyncMock(return_value="backup async response")

        failover = FailoverVLM(primary, backup)

        result = await failover.get_completion_async(prompt="test")

        assert result == "backup async response"
        primary.get_completion_async.assert_called_once()
        backup.get_completion_async.assert_called_once()

    @pytest.mark.asyncio
    async def test_async_vision_completion_failover(self):
        """Test failover works for async vision completion."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_vision_completion_async = AsyncMock(
            side_effect=Exception('API Error: 429 {"error":{"code":"AccountQuotaExceeded"}}')
        )

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_vision_completion_async = AsyncMock(return_value="backup async vision response")

        failover = FailoverVLM(primary, backup)

        result = await failover.get_vision_completion_async(prompt="describe", images=["test.jpg"])

        assert result == "backup async vision response"


class TestVLMConfigWithBackup:
    """Tests for VLMConfig with backup configuration integration."""

    async def test_account_config_updates_preserve_inflight_calls_and_usage(self, monkeypatch):
        import asyncio

        from openviking.config.binding import manager_over_source
        from openviking.config.source import MemoryConfigSource
        from openviking.config.vlm import AccountVLMProvider
        from openviking_cli.utils.config import set_openviking_config
        from openviking_cli.utils.config.open_viking_config import (
            OpenVikingConfig,
            OpenVikingConfigSingleton,
        )

        started, release = asyncio.Event(), asyncio.Event()
        clients = []

        class Client:
            def __init__(self, config):
                self.model = config["model"]
                self._token_tracker = TokenUsageTracker()
                self.closed = 0
                clients.append(self)

            async def get_completion_async(self, prompt, max_tokens, **kwargs):
                if prompt == "hold":
                    started.set()
                    await release.wait()
                self._token_tracker.update(self.model, "openai", 100, 20)
                return f"{self.model}:{max_tokens}"

            def close(self):
                self.closed += 1

        monkeypatch.setattr("openviking.models.vlm.VLMFactory.create", Client)
        base = OpenVikingConfig.from_dict(
            {"vlm": {"model": "cluster", "provider": "openai", "api_key": "cluster-key"}}
        )
        set_openviking_config(base)
        manager = manager_over_source(MemoryConfigSource(), base_config=base)
        provider = AccountVLMProvider(manager)
        pending = None
        try:
            await manager.initialize()
            await manager.patch_account(
                "a",
                {
                    "vlm": {
                        "model": "a-v1",
                        "credentials": [{"provider": "openai", "api_key": "a-key"}],
                    }
                },
            )
            account = await provider.get_vlm("a")
            other = await provider.get_vlm("b")
            assert await account.get_completion_async("first", max_tokens=64) == "a-v1:64"
            assert provider.get_token_usage("a")["total_usage"]["total_tokens"] == 120
            assert provider.get_token_usage("b")["total_usage"]["total_tokens"] == 0

            pending = asyncio.create_task(account.get_completion_async("hold", max_tokens=64))
            await started.wait()
            await manager.patch_account("a", {"vlm": {"model": "a-v2"}})
            assert clients[0].closed == 0
            assert await other.get_completion_async("other", max_tokens=32) == "cluster:32"
            pending.cancel()
            with pytest.raises(asyncio.CancelledError):
                await pending
            assert clients[0].closed == 1
            assert await account.get_completion_async("updated", max_tokens=16) == "a-v2:16"
            assert provider.get_token_usage("a")["total_usage"]["total_tokens"] == 240
            assert provider.get_token_usage("b")["total_usage"]["total_tokens"] == 120
        finally:
            release.set()
            if pending is not None:
                pending.cancel()
                await asyncio.gather(pending, return_exceptions=True)
            await provider.close()
            OpenVikingConfigSingleton.reset_instance()
        assert len(clients) == 3
        assert all(client.closed == 1 for client in clients)

    def test_config_with_backup_creates_multicredential_instance(self, monkeypatch):
        """Test that config with backup creates a MultiCredentialVLM instance."""
        mock_factory = Mock()
        mock_primary = Mock()
        mock_backup = Mock()
        mock_factory.create.side_effect = [mock_primary, mock_backup]

        monkeypatch.setattr("openviking.models.vlm.VLMFactory", mock_factory)

        backup_config = VLMConfig(
            model="backup-model",
            api_key="backup-key",
            provider="openai",
        )
        config = VLMConfig(
            model="primary-model",
            api_key="primary-key",
            provider="volcengine",
            backup=backup_config,
        )

        instance = config.get_vlm_instance()

        # Should be a MultiCredentialVLM instance
        assert hasattr(instance, "_vlm_instances")
        assert len(instance._vlm_instances) == 2


class TestPrimaryBackupSwitcher:
    """Tests for PrimaryBackupSwitcher."""

    def test_initial_state(self):
        """Test initial state is using primary."""
        switcher = PrimaryBackupSwitcher()
        assert switcher.is_using_backup is False
        assert switcher.should_try_primary() is True

    def test_record_primary_success_no_change(self):
        """Test successful primary calls keep us on primary."""
        switcher = PrimaryBackupSwitcher()
        switcher.record_primary_success()
        assert switcher.is_using_backup is False

    def test_switch_to_backup_on_permanent_error(self):
        """Test permanent error triggers switch to backup."""
        switcher = PrimaryBackupSwitcher()
        error = Exception("403 forbidden")
        switched = switcher.record_primary_failure(error)
        assert switched is True
        assert switcher.is_using_backup is True

    def test_switch_to_backup_on_quota_error(self):
        """Test quota error triggers switch to backup."""
        switcher = PrimaryBackupSwitcher()
        error = Exception("quota exceeded")
        switched = switcher.record_primary_failure(error)
        assert switched is True
        assert switcher.is_using_backup is True

    def test_no_switch_on_other_errors(self):
        """Test other errors don't trigger switch."""
        switcher = PrimaryBackupSwitcher()
        error = Exception("invalid prompt")
        switched = switcher.record_primary_failure(error)
        assert switched is False
        assert switcher.is_using_backup is False

    def test_should_try_primary_after_request_count(self):
        """Test failback attempt after enough backup requests."""
        switcher = PrimaryBackupSwitcher(failback_request_count=3)

        # Switch to backup
        error = Exception("quota exceeded")
        switcher.record_primary_failure(error)
        assert switcher.is_using_backup is True

        # Record backup requests
        assert switcher.should_try_primary() is False
        switcher.record_backup_request()
        assert switcher.should_try_primary() is False
        switcher.record_backup_request()
        assert switcher.should_try_primary() is False
        switcher.record_backup_request()
        assert switcher.should_try_primary() is True

    def test_should_try_primary_after_timeout(self):
        """Test failback attempt after timeout."""
        switcher = PrimaryBackupSwitcher(failback_timeout_seconds=0.1)

        # Switch to backup
        error = Exception("quota exceeded")
        switcher.record_primary_failure(error)
        assert switcher.is_using_backup is True

        # Before timeout
        assert switcher.should_try_primary() is False

        # After timeout
        time.sleep(0.15)
        assert switcher.should_try_primary() is True

    def test_failback_success_stays_on_primary(self):
        """Test successful failback keeps us on primary."""
        switcher = PrimaryBackupSwitcher(failback_request_count=1)

        # Switch to backup
        error = Exception("quota exceeded")
        switcher.record_primary_failure(error)
        switcher.record_backup_request()

        # Now try primary and succeed
        assert switcher.should_try_primary() is True
        switcher.record_primary_success()

        assert switcher.is_using_backup is False

    def test_failback_failure_goes_back_to_backup(self):
        """Test failed failback goes back to backup and resets counter."""
        switcher = PrimaryBackupSwitcher(failback_request_count=2)

        # Switch to backup
        error = Exception("quota exceeded")
        switcher.record_primary_failure(error)
        switcher.record_backup_request()
        switcher.record_backup_request()

        # Now try primary and fail again
        assert switcher.should_try_primary() is True
        switched = switcher.record_primary_failure(Exception("quota exceeded again"))

        assert switched is True
        assert switcher.is_using_backup is True

        # Counter should be reset
        switcher.record_backup_request()
        assert switcher.should_try_primary() is False


class TestFailoverVLMAutomaticFailback:
    """Tests for FailoverVLM automatic failback functionality."""

    def test_failback_after_request_count(self):
        """Test automatic failback after enough backup requests."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        # First call fails, then succeeds
        primary.get_completion.side_effect = [
            Exception("quota exceeded"),
            "primary is back!",
        ]

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.return_value = "backup response"

        failover = FailoverVLM(primary, backup, failback_request_count=2)

        # First call - fails over to backup
        result1 = failover.get_completion(prompt="test1")
        assert result1 == "backup response"
        assert failover.is_using_backup is True

        # Second call - still using backup
        result2 = failover.get_completion(prompt="test2")
        assert result2 == "backup response"

        # Third call - should try primary again and succeed
        result3 = failover.get_completion(prompt="test3")
        assert result3 == "primary is back!"
        assert failover.is_using_backup is False

    def test_failback_failure_stays_on_backup(self):
        """Test if failback fails, it stays on backup and resets counter."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        # Always fails
        primary.get_completion.side_effect = Exception("quota exceeded")

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.return_value = "backup response"

        failover = FailoverVLM(primary, backup, failback_request_count=2)

        # First call - fails over
        result1 = failover.get_completion(prompt="test1")
        assert result1 == "backup response"

        # Second call - still on backup
        result2 = failover.get_completion(prompt="test2")
        assert result2 == "backup response"

        # Third call - tries primary, fails again, goes back to backup
        result3 = failover.get_completion(prompt="test3")
        assert result3 == "backup response"
        assert failover.is_using_backup is True

        # Counter should be reset - next call doesn't try primary yet
        result4 = failover.get_completion(prompt="test4")
        assert result4 == "backup response"

    def test_failback_after_timeout(self):
        """Test automatic failback after timeout."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion.side_effect = [
            Exception("quota exceeded"),
            "primary is back!",
        ]

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion.return_value = "backup response"

        failover = FailoverVLM(primary, backup, failback_timeout_seconds=0.1)

        # First call - fails over
        result1 = failover.get_completion(prompt="test1")
        assert result1 == "backup response"
        assert failover.is_using_backup is True

        # Wait for timeout
        time.sleep(0.15)

        # Next call - should try primary again
        result2 = failover.get_completion(prompt="test2")
        assert result2 == "primary is back!"
        assert failover.is_using_backup is False

    @pytest.mark.asyncio
    async def test_async_failback_after_request_count(self):
        """Test automatic failback works with async methods."""
        primary = Mock()
        primary.model = "primary-model"
        primary.provider = "volcengine"
        primary.get_completion_async = AsyncMock(
            side_effect=[
                Exception("quota exceeded"),
                "primary async is back!",
            ]
        )

        backup = Mock()
        backup.model = "backup-model"
        backup.provider = "openai"
        backup.get_completion_async = AsyncMock(return_value="backup async response")

        failover = FailoverVLM(primary, backup, failback_request_count=2)

        # First call - fails over
        result1 = await failover.get_completion_async(prompt="test1")
        assert result1 == "backup async response"

        # Second call - still on backup
        result2 = await failover.get_completion_async(prompt="test2")
        assert result2 == "backup async response"

        # Third call - tries primary and succeeds
        result3 = await failover.get_completion_async(prompt="test3")
        assert result3 == "primary async is back!"
        assert failover.is_using_backup is False