Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
14.5 KB
Raw Blame
"""Tests for default model selection when a provider is removed.

When a provider is removed and it was the active LLM or embedding provider,
the backend should fall back to another configured provider AND select a
sensible default model (not an empty string).

Ollama is the fallback these cases exercise, and a fallback helper will not
hand back a provider the run mode hides. The module pins a run mode that
offers Ollama so these stay about fallback selection whatever
`config/model_providers.yaml` says; `test_model_providers_config` covers the
hiding itself.
"""

import pytest

from api.settings.helpers import (
    _default_embedding_model,
    _default_llm_model,
    _first_configured_embedding_provider,
    _first_configured_llm_provider,
    _has_other_configured_provider,
)
from config.config_manager import (
    AgentConfig,
    AnthropicConfig,
    KnowledgeConfig,
    OllamaConfig,
    OnboardingState,
    OpenAIConfig,
    OpenRAGConfig,
    ProvidersConfig,
    WatsonXConfig,
)
from config.model_constants import (
    ANTHROPIC_DEFAULT_LANGUAGE_MODEL,
    OPENAI_DEFAULT_LANGUAGE_MODEL,
)

# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture(autouse=True)
def _run_mode_that_offers_ollama(monkeypatch):
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")


def _make_config(
    *,
    openai=False,
    anthropic=False,
    ollama=False,
    watsonx=False,
    llm_provider="openai",
    llm_model="gpt-5.4-mini",
    embedding_provider="openai",
    embedding_model="text-embedding-3-small",
) -> OpenRAGConfig:
    """Build a minimal OpenRAGConfig with the requested providers configured."""
    return OpenRAGConfig(
        providers=ProvidersConfig(
            openai=OpenAIConfig(api_key="sk-test" if openai else "", configured=openai),
            anthropic=AnthropicConfig(api_key="sk-ant" if anthropic else "", configured=anthropic),
            ollama=OllamaConfig(
                endpoint="http://localhost:11434" if ollama else "", configured=ollama
            ),
            watsonx=WatsonXConfig(
                api_key="wx-key" if watsonx else "",
                endpoint="https://us-south.ml.cloud.ibm.com" if watsonx else "",
                project_id="pid" if watsonx else "",
                configured=watsonx,
            ),
        ),
        knowledge=KnowledgeConfig(
            embedding_model=embedding_model,
            embedding_provider=embedding_provider,
        ),
        agent=AgentConfig(
            llm_model=llm_model,
            llm_provider=llm_provider,
        ),
        onboarding=OnboardingState(),
        edited=True,
    )


# ---------------------------------------------------------------------------
# _default_llm_model
# ---------------------------------------------------------------------------


class TestDefaultLlmModel:
    def test_openai_returns_static_default(self):
        assert _default_llm_model("openai") == OPENAI_DEFAULT_LANGUAGE_MODEL

    def test_anthropic_returns_static_default(self):
        assert _default_llm_model("anthropic") == ANTHROPIC_DEFAULT_LANGUAGE_MODEL

    def test_ollama_returns_empty(self):
        assert _default_llm_model("ollama") == ""

    def test_watsonx_returns_empty(self):
        assert _default_llm_model("watsonx") == ""

    def test_azure_returns_empty(self):
        assert _default_llm_model("azure") == ""

    def test_unknown_provider_returns_empty(self):
        assert _default_llm_model("nonexistent") == ""


# ---------------------------------------------------------------------------
# _default_embedding_model
# ---------------------------------------------------------------------------


class TestDefaultEmbeddingModel:
    def test_openai_returns_empty(self):
        """ "openai" often means an internal OpenAI-compatible gateway with a
        curated model set — never guess, force an explicit, validated pick."""
        assert _default_embedding_model("openai") == ""

    def test_ollama_returns_empty(self):
        assert _default_embedding_model("ollama") == ""

    def test_watsonx_returns_empty(self):
        assert _default_embedding_model("watsonx") == ""

    def test_unknown_provider_returns_empty(self):
        assert _default_embedding_model("nonexistent") == ""

    def test_uses_deployment_declared_default_when_set(self, monkeypatch):
        """When the deployment declares EMBEDDING_MODEL/EMBEDDING_PROVIDER
        (Helm values / operator ConfigMap), the fallback should use it
        instead of returning empty — this is what lets a correctly
        configured deployment self-heal after a provider removal."""
        monkeypatch.setenv("EMBEDDING_PROVIDER", "openai")
        monkeypatch.setenv("EMBEDDING_MODEL", "text-embedding-3-large")
        assert _default_embedding_model("openai") == "text-embedding-3-large"

    def test_ignores_declared_default_for_a_different_provider(self, monkeypatch):
        """A declared default for "openai" must not leak into the fallback
        for a provider it wasn't declared for."""
        monkeypatch.setenv("EMBEDDING_PROVIDER", "openai")
        monkeypatch.setenv("EMBEDDING_MODEL", "text-embedding-3-large")
        assert _default_embedding_model("watsonx") == ""

    def test_azure_requires_an_explicit_deployment_even_when_env_declares_one(self, monkeypatch):
        """Azure model names are deployment names chosen by the customer.

        Provider removal must not silently carry an operator default into the
        user's workspace; both Azure model choices are made explicitly.
        """
        monkeypatch.setenv("EMBEDDING_PROVIDER", "azure")
        monkeypatch.setenv("EMBEDDING_MODEL", "text-embedding-3-small")
        assert _default_embedding_model("azure") == ""


# ---------------------------------------------------------------------------
# _first_configured_llm_provider
# ---------------------------------------------------------------------------


class TestFirstConfiguredLlmProvider:
    def test_excludes_removed_provider(self):
        config = _make_config(openai=True, anthropic=True)
        assert _first_configured_llm_provider(config, "openai") == "anthropic"

    def test_respects_priority_order(self):
        """Order is openai > anthropic > watsonx > ollama."""
        config = _make_config(openai=True, anthropic=True, watsonx=True, ollama=True)
        assert _first_configured_llm_provider(config, "openai") == "anthropic"
        assert _first_configured_llm_provider(config, "anthropic") == "openai"

    def test_skips_unconfigured(self):
        config = _make_config(ollama=True)
        assert _first_configured_llm_provider(config, "openai") == "ollama"

    def test_falls_back_to_openai_when_none_configured(self):
        config = _make_config()
        assert _first_configured_llm_provider(config, "anthropic") == "openai"


# ---------------------------------------------------------------------------
# _first_configured_embedding_provider
# ---------------------------------------------------------------------------


class TestFirstConfiguredEmbeddingProvider:
    def test_excludes_removed_provider(self):
        config = _make_config(openai=True, ollama=True)
        assert _first_configured_embedding_provider(config, "openai") == "ollama"

    def test_anthropic_never_returned(self):
        """Anthropic has no embedding models; it must never be a fallback."""
        config = _make_config(anthropic=True, ollama=True)
        assert _first_configured_embedding_provider(config, "openai") == "ollama"

    def test_respects_priority_order(self):
        """Order is openai > watsonx > ollama."""
        config = _make_config(openai=True, watsonx=True, ollama=True)
        assert _first_configured_embedding_provider(config, "openai") == "watsonx"

    def test_returns_empty_when_none_configured(self):
        config = _make_config()
        assert _first_configured_embedding_provider(config, "ollama") == ""


# ---------------------------------------------------------------------------
# Simulated provider removal: LLM model default
# ---------------------------------------------------------------------------


class TestProviderRemovalLlmDefault:
    """Simulate the provider-removal code path from endpoints.py and verify
    that the resulting llm_model is a sensible default, not empty."""

    def _simulate_llm_removal(self, config: OpenRAGConfig, removed: str):
        """Replicate the fallback logic from update_settings()."""
        if config.agent.llm_provider == removed:
            fb = _first_configured_llm_provider(config, removed)
            config.agent.llm_provider = fb
            config.agent.llm_model = _default_llm_model(fb)

    def test_remove_anthropic_falls_back_to_openai_model(self):
        config = _make_config(
            openai=True,
            anthropic=True,
            llm_provider="anthropic",
            llm_model="claude-sonnet-4-6",
        )
        self._simulate_llm_removal(config, "anthropic")
        assert config.agent.llm_provider == "openai"
        assert config.agent.llm_model == OPENAI_DEFAULT_LANGUAGE_MODEL

    def test_remove_openai_falls_back_to_anthropic_model(self):
        config = _make_config(
            openai=True,
            anthropic=True,
            llm_provider="openai",
            llm_model="gpt-5.4-mini",
        )
        self._simulate_llm_removal(config, "openai")
        assert config.agent.llm_provider == "anthropic"
        assert config.agent.llm_model == ANTHROPIC_DEFAULT_LANGUAGE_MODEL

    def test_remove_openai_falls_back_to_ollama_empty_model(self):
        """Ollama models are dynamic — backend returns empty, frontend picks."""
        config = _make_config(
            openai=True,
            ollama=True,
            llm_provider="openai",
            llm_model="gpt-5.4-mini",
        )
        self._simulate_llm_removal(config, "openai")
        assert config.agent.llm_provider == "ollama"
        assert config.agent.llm_model == ""

    def test_remove_watsonx_falls_back_to_openai_model(self):
        config = _make_config(
            openai=True,
            watsonx=True,
            llm_provider="watsonx",
            llm_model="ibm/granite-13b-chat-v2",
        )
        self._simulate_llm_removal(config, "watsonx")
        assert config.agent.llm_provider == "openai"
        assert config.agent.llm_model == OPENAI_DEFAULT_LANGUAGE_MODEL

    def test_no_change_if_different_provider_removed(self):
        """If the removed provider wasn't the active one, nothing changes."""
        config = _make_config(
            openai=True,
            anthropic=True,
            llm_provider="openai",
            llm_model="gpt-5.4-mini",
        )
        self._simulate_llm_removal(config, "anthropic")
        assert config.agent.llm_provider == "openai"
        assert config.agent.llm_model == "gpt-5.4-mini"


# ---------------------------------------------------------------------------
# Simulated provider removal: embedding model default
# ---------------------------------------------------------------------------


class TestProviderRemovalEmbeddingDefault:
    """Simulate embedding provider fallback on removal."""

    def _simulate_embedding_removal(self, config: OpenRAGConfig, removed: str):
        if config.knowledge.embedding_provider == removed:
            fb = _first_configured_embedding_provider(config, removed)
            config.knowledge.embedding_provider = fb
            config.knowledge.embedding_model = _default_embedding_model(fb)

    def test_remove_ollama_falls_back_to_openai_empty_embedding(self):
        """OpenAI's embedding catalog isn't guessed — the admin must pick
        one the settings UI confirms is actually available."""
        config = _make_config(
            openai=True,
            ollama=True,
            embedding_provider="ollama",
            embedding_model="nomic-embed-text",
        )
        self._simulate_embedding_removal(config, "ollama")
        assert config.knowledge.embedding_provider == "openai"
        assert config.knowledge.embedding_model == ""

    def test_remove_openai_falls_back_to_watsonx_empty_embedding(self):
        config = _make_config(
            openai=True,
            watsonx=True,
            embedding_provider="openai",
            embedding_model="text-embedding-3-small",
        )
        self._simulate_embedding_removal(config, "openai")
        assert config.knowledge.embedding_provider == "watsonx"
        assert config.knowledge.embedding_model == ""

    def test_remove_watsonx_falls_back_to_openai_empty_embedding(self):
        """Same as above: watsonx -> openai fallback must not guess a model."""
        config = _make_config(
            openai=True,
            watsonx=True,
            embedding_provider="watsonx",
            embedding_model="ibm/slate-125m-english-rtrvr",
        )
        self._simulate_embedding_removal(config, "watsonx")
        assert config.knowledge.embedding_provider == "openai"
        assert config.knowledge.embedding_model == ""

    def test_no_change_if_different_provider_removed(self):
        config = _make_config(
            openai=True,
            ollama=True,
            embedding_provider="openai",
            embedding_model="text-embedding-3-small",
        )
        self._simulate_embedding_removal(config, "ollama")
        assert config.knowledge.embedding_provider == "openai"
        assert config.knowledge.embedding_model == "text-embedding-3-small"


class TestHasOtherConfiguredProvider:
    def test_single_provider_cannot_be_removed(self):
        config = _make_config(watsonx=True)
        assert not _has_other_configured_provider(config, "watsonx")

    def test_cannot_remove_provider_when_only_anthropic_is_configured(self):
        config = _make_config(watsonx=True, anthropic=True)
        assert not _has_other_configured_provider(config, "watsonx")

    def test_can_remove_anthropic_when_watsonx_is_configured(self):
        config = _make_config(watsonx=True, anthropic=True)
        assert _has_other_configured_provider(config, "anthropic")

    def test_can_remove_watsonx_when_azure_custom_provider_is_configured(self):
        from config.config_manager import GenericProviderConfig

        config = _make_config(watsonx=True)
        config.providers.custom["azure"] = GenericProviderConfig(configured=True)
        assert _has_other_configured_provider(config, "watsonx")
        assert _has_other_configured_provider(config, "azure")

    def test_can_remove_watsonx_when_openai_is_configured(self):
        config = _make_config(watsonx=True, openai=True)
        assert _has_other_configured_provider(config, "watsonx")