Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
25.4 KB
Raw Blame
"""The LiteLLM-derived model catalogue.

The catalogue replaced per-provider live `/models/{provider}` fetches for the
settings dropdown, so these tests pin the properties that table used to
guarantee: a picked model names a provider LiteLLM can call, chat and embedding
lists are disjoint, and the OpenAI-compatible /v1/models list is derived from
the same payload.
"""

from __future__ import annotations

from types import SimpleNamespace

import pytest

from config import model_providers
from services import model_catalog


@pytest.fixture(autouse=True)
def _fresh_provider_config():
    """Run-mode tests below rewrite the provider config; don't leak the cache."""
    model_providers.reload()
    yield
    model_providers.reload()


def test_catalog_only_groups_supported_providers_with_their_models_and_forms() -> None:
    providers = {entry["key"]: entry for entry in model_catalog.catalog()["providers"]}

    assert set(providers) == model_catalog.supported_provider_keys()
    for key in model_catalog.supported_provider_keys():
        assert key in providers, key
        assert providers[key]["credential_fields"], key

    openai = providers["openai"]
    assert openai["name"] == "OpenAI"
    assert any(entry["model"].startswith("gpt-") for entry in openai["models"])
    assert any("embedding" in entry["model"] for entry in openai["embedding_models"])
    field_types = {field["key"]: field["field_type"] for field in openai["credential_fields"]}
    assert field_types["api_key"] == "password"
    assert field_types["api_base"] == "text"


def test_catalog_chat_models_are_only_text_generation() -> None:
    modes = {
        entry["mode"]
        for provider in model_catalog.catalog()["providers"]
        for entry in provider["models"]
    }
    assert modes <= model_catalog.TEXT_GENERATION_MODES
    assert "chat" in modes


def test_catalog_embedding_models_are_only_embedding_mode() -> None:
    modes = {
        entry["mode"]
        for provider in model_catalog.catalog()["providers"]
        for entry in provider["embedding_models"]
    }
    assert modes <= {model_catalog.EMBEDDING_MODE}
    assert model_catalog.EMBEDDING_MODE in modes


def test_model_ids_are_stored_without_their_provider_prefix() -> None:
    providers = {entry["key"]: entry for entry in model_catalog.catalog()["providers"]}
    for provider, entry in providers.items():
        for model in entry["models"] + entry["embedding_models"]:
            assert not model["model"].startswith(f"{provider}/")


def test_openai_form_is_the_plain_one_not_the_compatible_variant() -> None:
    required = model_catalog.required_field_keys("openai")
    assert required == ["api_key"]


def test_azure_openai_leaves_auth_fields_optional_for_method_selection() -> None:
    """The UI selects one complete Azure auth method over LiteLLM's flat form."""
    assert model_catalog.required_field_keys("azure") == ["api_base"]


def test_an_unknown_provider_still_gets_a_usable_form() -> None:
    fields = model_catalog.credential_fields("some-private-gateway")
    assert [field["key"] for field in fields] == ["api_key", "api_base"]
    assert model_catalog.missing_required_fields("some-private-gateway", set()) == []


def test_a_tls_verify_field_keeps_its_type_and_is_not_a_secret() -> None:
    """The UI renders `tls_verify` as a switch, so normalizing must not turn it into text."""
    fields = {field["key"]: field for field in model_catalog.credential_fields("rhoai")}
    assert fields["ssl_verify"]["field_type"] == "tls_verify"
    assert "ssl_verify" not in model_catalog.secret_field_keys("rhoai")


def test_an_unrecognised_field_type_falls_back_to_text() -> None:
    field = model_catalog._normalize_field({"key": "x", "field_type": "colour"})
    assert field["field_type"] == "text"


def test_openai_models_list_is_openai_compatible() -> None:
    payload = model_catalog.openai_models_list()
    assert payload["object"] == "list"
    assert payload["data"]
    ids = {row["id"] for row in payload["data"]}
    assert any(model_id.startswith("gpt-") for model_id in ids)
    sample = payload["data"][0]
    assert sample["object"] == "model"
    assert "owned_by" in sample


def test_is_known_provider_accepts_litellm_handlers() -> None:
    assert model_catalog.is_known_provider("openai")
    assert model_catalog.is_known_provider("anthropic")
    assert not model_catalog.is_known_provider("not-a-real-provider")
    assert not model_catalog.is_known_provider("")


def test_is_known_provider_covers_every_provider_litellm_can_route() -> None:
    """`provider_list` is enum members, not strings.

    Reading them with str() gives "LlmProviders.OPENAI", so the check silently
    fell through to the credential-form specs — which omit ~26 routable
    providers. Their `provider/model` ids were then left unsplit and billed to
    whatever the default provider was.
    """
    import litellm

    routable = {getattr(value, "value", str(value)) for value in litellm.provider_list}
    assert routable, "litellm exposes no provider list"
    missing = sorted(key for key in routable if not model_catalog.is_known_provider(key))
    assert not missing, missing


def test_a_routable_prefix_without_a_credential_form_still_splits() -> None:
    from services.llm_gateway import split_model_id

    assert split_model_id("zai/glm-4.6") == ("zai", "glm-4.6")
    assert split_model_id("not-a-provider/x") == (None, "not-a-provider/x")


def test_exported_model_ids_route_back_to_their_owner() -> None:
    """A `/v1/models` id must resolve to the provider that owns it.

    `_catalog()` strips provider prefixes for the picker; without re-attaching
    them here an Anthropic id sent to `/v1/chat/completions` would fall through
    to the default provider and be called with the wrong credentials.
    """
    from services.llm_gateway import split_model_id

    payload = model_catalog.openai_models_list()
    non_openai = [row for row in payload["data"] if row["owned_by"] != "openai"]
    assert non_openai, "catalogue should expose more than OpenAI"

    for row in non_openai:
        provider, _name = split_model_id(row["id"])
        assert provider == row["owned_by"], row

    openai_rows = [row for row in payload["data"] if row["owned_by"] == "openai"]
    assert openai_rows
    assert all("/" not in row["id"] for row in openai_rows)


def test_catalog_hides_a_provider_the_run_mode_does_not_expose(monkeypatch) -> None:
    """SaaS must not advertise Ollama; oss and on-prem must keep it."""
    monkeypatch.setenv("OPENRAG_RUN_MODE", "saas")
    saas = {entry["key"] for entry in model_catalog.catalog()["providers"]}
    assert "ollama" not in saas
    assert "openai" in saas

    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    assert "ollama" in {entry["key"] for entry in model_catalog.catalog()["providers"]}


def test_catalog_publishes_azure_ai_where_the_config_enables_it(monkeypatch) -> None:
    for mode in ("oss", "on_prem", "saas"):
        monkeypatch.setenv("OPENRAG_RUN_MODE", mode)
        providers = {entry["key"]: entry for entry in model_catalog.catalog()["providers"]}
        assert "azure_ai" in providers, mode
        azure = providers["azure_ai"]
        assert azure["name"] == "Azure AI Foundry"
        assert azure["models"], mode
        assert azure["embedding_models"], mode
        assert {"api_base", "api_key"} <= {field["key"] for field in azure["credential_fields"]}


def test_azure_openai_is_the_row_that_carries_the_gpt_41_family(monkeypatch) -> None:
    """The two Azure rows are not interchangeable.

    LiteLLM files Azure OpenAI Service under `azure` and the Foundry catalogue
    under `azure_ai`, and only the former lists gpt-4.1. Offering just
    `azure_ai` — as the config first did — leaves no way to pick gpt-4.1 from
    the dropdown at all.
    """
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    providers = {entry["key"]: entry for entry in model_catalog.catalog()["providers"]}

    assert providers["azure"]["name"] == "Azure OpenAI"
    assert "gpt-4.1" in {entry["model"] for entry in providers["azure"]["models"]}
    assert "gpt-4.1" not in {entry["model"] for entry in providers["azure_ai"]["models"]}
    # The prefix is stripped for display but re-attached on the way out, or the
    # id would be called with the default provider's credentials.
    assert "azure:gpt-4.1" in {row["id"] for row in model_catalog.openai_models_list()["data"]}


def test_openai_models_list_drops_a_hidden_providers_models(monkeypatch) -> None:
    monkeypatch.setenv("OPENRAG_RUN_MODE", "saas")
    owners = {row["owned_by"] for row in model_catalog.openai_models_list()["data"]}
    assert "ollama" not in owners

    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    assert "ollama" in {row["owned_by"] for row in model_catalog.openai_models_list()["data"]}


def test_the_config_display_name_names_the_provider(monkeypatch, tmp_path) -> None:
    """A catalogue entry is labelled by the config file, not by LiteLLM."""
    path = tmp_path / "model_providers.yaml"
    path.write_text(
        "providers:\n  - name: openai\n    display_name: House Gateway\n"
        "    modes:\n      oss: true\n",
        encoding="utf-8",
    )
    monkeypatch.setenv(model_providers.CONFIG_PATH_ENV, str(path))
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    providers = model_catalog.catalog()["providers"]
    assert [entry["key"] for entry in providers] == ["openai"]
    assert providers[0]["name"] == "House Gateway"


def _write_providers(tmp_path, body: str) -> str:
    path = tmp_path / "model_providers.yaml"
    path.write_text(body, encoding="utf-8")
    return str(path)


def test_a_custom_gateway_can_declare_the_models_it_serves(monkeypatch, tmp_path) -> None:
    """LiteLLM's table has no ids for a self-hosted OpenAI-compatible endpoint.

    Without config-declared ids the provider would render a card and a working
    credential form, then offer an empty model picker.
    """
    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            """
providers:
  - name: openai_like
    display_name: Internal LLM Gateway
    modes:
      oss: true
    models:
      - llama-3.3-70b-instruct
    embedding_models:
      - bge-m3
""",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    providers = model_catalog.catalog()["providers"]
    assert [entry["key"] for entry in providers] == ["openai_like"]
    gateway = providers[0]
    assert gateway["name"] == "Internal LLM Gateway"
    assert [entry["model"] for entry in gateway["models"]] == ["llama-3.3-70b-instruct"]
    assert [entry["model"] for entry in gateway["embedding_models"]] == ["bge-m3"]
    # The endpoint has to be enterable, or the gateway is unreachable.
    assert "api_base" in {field["key"] for field in gateway["credential_fields"]}


def test_a_declared_gateway_model_routes_back_to_its_own_provider(monkeypatch, tmp_path) -> None:
    from services.llm_gateway import PROVIDER_SEPARATOR, split_model_id

    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            "providers:\n  - name: openai_like\n    modes:\n      oss: true\n"
            "    models:\n      - llama-3.3-70b-instruct\n",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    ids = [row["id"] for row in model_catalog.openai_models_list()["data"]]
    assert ids == [f"openai_like{PROVIDER_SEPARATOR}llama-3.3-70b-instruct"]
    assert split_model_id(ids[0]) == ("openai_like", "llama-3.3-70b-instruct")


def test_declared_ids_do_not_duplicate_what_litellm_already_lists(monkeypatch, tmp_path) -> None:
    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            "providers:\n  - name: openai\n    modes:\n      oss: true\n"
            "    models:\n      - gpt-4o\n      - house-tuned-gpt\n",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    models = [entry["model"] for entry in model_catalog.catalog()["providers"][0]["models"]]
    assert models.count("gpt-4o") == 1
    assert "house-tuned-gpt" in models


def test_a_configured_exclusion_keeps_models_out_of_the_catalogue(monkeypatch, tmp_path) -> None:
    """`exclude_models` is how a deployment refuses a model outright.

    Distinct from the two automatic paths: a retired model is dropped without
    being named here, and one retiring soon is only hidden behind a toggle in
    the picker.
    """
    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            "providers:\n  - name: openai\n    modes:\n      oss: true\n"
            "    exclude_models:\n      - gpt-3.5-*\n      - gpt-4o\n",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    models = {entry["model"] for entry in model_catalog.catalog()["providers"][0]["models"]}
    assert not any(name.startswith("gpt-3.5") for name in models), sorted(models)[:5]
    assert "gpt-4o" not in models
    # Only what was named: a wildcard must not take the neighbours with it.
    assert "gpt-4o-mini" in models
    assert any(name.startswith("gpt-4.1") for name in models)


def test_an_exclusion_also_overrides_a_declared_model(monkeypatch, tmp_path) -> None:
    """Two rows that contradict each other resolve one way, predictably."""
    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            "providers:\n  - name: openai\n    modes:\n      oss: true\n"
            "    models:\n      - house-tuned-gpt\n"
            "    exclude_models:\n      - house-tuned-*\n",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    models = {entry["model"] for entry in model_catalog.catalog()["providers"][0]["models"]}
    assert "house-tuned-gpt" not in models


def test_an_excluded_model_is_gone_from_v1_models_too(monkeypatch, tmp_path) -> None:
    """The picker and the OpenAI-compatible list must not disagree."""
    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            "providers:\n  - name: openai\n    modes:\n      oss: true\n"
            "    exclude_models:\n      - gpt-4o\n",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    assert "gpt-4o" not in {row["id"] for row in model_catalog.openai_models_list()["data"]}


def test_exclusions_reach_the_live_provider_lists_too(monkeypatch, tmp_path) -> None:
    """Ollama, watsonx, OpenAI and Anthropic are also listed by asking the provider.

    That path never touches the catalogue, so without filtering it an excluded
    id vanishes from Settings and walks straight back in through onboarding.
    """
    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            "providers:\n  - name: openai\n    modes:\n      oss: true\n"
            "    exclude_models:\n      - gpt-3.5-*\n",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    payload = {
        "language_models": [
            {"value": "gpt-3.5-turbo", "label": "gpt-3.5-turbo"},
            {"value": "gpt-4o", "label": "gpt-4o"},
        ],
        "embedding_models": [{"value": "text-embedding-3-small"}],
    }
    filtered = model_catalog.hide_excluded_live_models("openai", payload)

    assert [entry["value"] for entry in filtered["language_models"]] == ["gpt-4o"]
    assert [entry["value"] for entry in filtered["embedding_models"]] == ["text-embedding-3-small"]
    # The caller's payload is not mutated; only the response is filtered.
    assert len(payload["language_models"]) == 2


def test_a_provider_with_no_exclusions_is_passed_through_untouched(monkeypatch) -> None:
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    payload = {"language_models": [{"value": "llama3"}]}
    assert model_catalog.hide_excluded_live_models("ollama", payload) is payload


def test_a_live_payload_that_is_not_a_dict_survives(monkeypatch) -> None:
    """The live endpoints can return an error body; filtering must not raise."""
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    assert model_catalog.hide_excluded_live_models("openai", None) is None
    assert model_catalog.hide_excluded_live_models("openai", []) == []


def test_the_shipped_config_keeps_the_legacy_openai_generation_out(monkeypatch) -> None:
    """The exclusions the shipped file actually carries."""
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    models = {
        entry["model"]
        for provider in model_catalog.catalog()["providers"]
        if provider["key"] == "openai"
        for entry in provider["models"]
    }

    assert not any(name.startswith("gpt-3.5") for name in models)
    assert "gpt-5.6-luna" not in models
    # Everything else the generation filter must not have taken with it.
    assert "gpt-4o" in models
    assert any(name.startswith("gpt-5.6") for name in models)


def test_the_shipped_config_keeps_broken_watsonx_models_out_of_live_inventory(
    monkeypatch,
) -> None:
    """Undocumented or broken watsonx.ai ids must not be offered in its picker."""
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    excluded = [
        "bigscience/mt0-xxl-13b",
        "google/flan-t5-xl-3b",
        "ibm/granite-guardian-3-2-2b",
        "ibm/granite-guardian-3-3-8b",
        "meta-llama/llama-4-maverick-17b",
        "mistralai/mistral-small-2503",
        "mistralai/pixtral-12b-2409",
        "mistralai/mistral-medium-2505",
        "ibm/granite-3-3-8b-instruct",
        "ibm/granite-3-8b-instruct",
        "meta-llama/llama-3-2-90b-vision-instruct",
        "meta-llama/llama-3-2-11b-vision-instruct",
        "meta-llama/llama-3-2-3b-instruct",
        "meta-llama/llama-3-2-1b-instruct",
        "mistralai/mistral-large",
        "meta-llama/llama-guard-3-11b-vision",
        "ibm/granite-vision-3-2-2b",
        "ibm/granite-ttm-1536-96-r2",
        "ibm/granite-ttm-1024-96-r2",
        "ibm/granite-ttm-512-96-r2",
        "core42/jais-13b-chat",
        "ibm/granite-13b-chat-v2",
        "ibm/granite-13b-instruct-v2",
        "sdaia/allam-1-13b-instruct",
        "bigscience/mt0-xxl",
    ]
    retained = [
        "google/flan-t5-xl",
        "ibm/granite-guardian-3-2b",
        "ibm/granite-guardian-3-8b",
        "meta-llama/llama-4-maverick-17b-128e-instruct-fp8",
        "mistralai/mistral-small-3-1-24b-instruct-2503",
        "mistralai/pixtral-12b",
        "ibm/granite-4-h-small",
    ]
    payload = {
        "language_models": [{"value": model} for model in excluded + retained],
        "embedding_models": [{"value": "ibm/slate-125m-english-rtrvr"}],
    }

    filtered = model_catalog.hide_excluded_live_models("watsonx", payload)

    assert [entry["value"] for entry in filtered["language_models"]] == retained
    assert filtered["embedding_models"] == payload["embedding_models"]
    catalogued = {
        entry["model"]
        for provider in model_catalog.catalog()["providers"]
        if provider["key"] == "watsonx"
        for entry in provider["models"]
    }
    assert not catalogued.intersection(excluded)


def test_a_configured_vision_model_is_tagged_vision(monkeypatch, tmp_path) -> None:
    """`vision_models` adds the capability the VLM picker filters on.

    Applies to LiteLLM's own rows and to declared ones alike, and leaves every
    other model's capabilities as the table had them.
    """
    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            "providers:\n  - name: openai\n    modes:\n      oss: true\n"
            "    models:\n      - house-tuned-gpt\n"
            "    vision_models:\n      - gpt-5.5-cyber\n      - house-tuned-*\n",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    models = {entry["model"]: entry for entry in model_catalog.catalog()["providers"][0]["models"]}
    assert "vision" in models["gpt-5.5-cyber"]["capabilities"]
    assert "reasoning" in models["gpt-5.5-cyber"]["capabilities"]
    assert models["house-tuned-gpt"]["capabilities"] == ["vision"]
    assert "vision" not in models["gpt-5.5-cyber-preview"].get("capabilities", [])


def test_the_shipped_config_offers_watsonx_vision_models(monkeypatch) -> None:
    """The VLM picker shows only `vision` models, so watsonx needs some.

    Every watsonx id LiteLLM flags as vision is withdrawn and excluded; the
    ones watsonx.ai serves with image input are tagged by the config instead.
    """
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    vision = {
        entry["model"]
        for provider in model_catalog.catalog()["providers"]
        if provider["key"] == "watsonx"
        for entry in provider["models"]
        if "vision" in entry.get("capabilities", [])
    }
    assert vision == {
        "meta-llama/llama-4-maverick-17b-128e-instruct-fp8",
        "mistralai/mistral-small-3-1-24b-instruct-2503",
    }


def test_a_plain_name_also_excludes_its_regional_listings(monkeypatch, tmp_path) -> None:
    """Azure lists the same model per region; suppressing it means all of them."""
    monkeypatch.setenv(
        model_providers.CONFIG_PATH_ENV,
        _write_providers(
            tmp_path,
            "providers:\n  - name: azure\n    modes:\n      oss: true\n"
            "    exclude_models:\n      - gpt-5.6-luna\n",
        ),
    )
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")

    models = {entry["model"] for entry in model_catalog.catalog()["providers"][0]["models"]}
    assert not any(name.endswith("gpt-5.6-luna") for name in models), sorted(
        name for name in models if "luna" in name
    )
    # A region's other models are untouched.
    assert any(name.startswith("eu/") for name in models)


def test_the_shipped_config_excludes_on_azure_as_well_as_openai(monkeypatch) -> None:
    """The same model is served by two rows, so it takes two exclusions."""
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    ids = {row["id"] for row in model_catalog.openai_models_list()["data"]}

    assert not [model_id for model_id in ids if "gpt-3.5" in model_id or "luna" in model_id]


def test_an_exclusion_without_a_wildcard_leaves_the_dated_snapshots(monkeypatch) -> None:
    """Why the shipped luna patterns end in `*`.

    A pattern with no wildcard matches that id alone, so the provider's dated
    snapshot of the same model stays in the picker — which is how
    `gpt-5.6-luna-2026-07-09` survived an exclusion of `gpt-5.6-luna`. Pinned
    because the fix is a single character in a YAML file and reads like a typo.
    """
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    exact = model_catalog._matches("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna",))
    wildcard = model_catalog._matches("gpt-5.6-luna-2026-07-09", ("gpt-5.6-luna*",))

    assert not exact
    assert wildcard
    # The wildcard keeps covering the regional forms the bare name covered.
    assert model_catalog._matches("us/gpt-5.6-luna", ("gpt-5.6-luna*",))


def test_both_luna_generations_are_excluded_from_both_rows(monkeypatch) -> None:
    """LiteLLM's table cannot be trusted for this, so the config carries it.

    Every luna row reports `supports_function_calling: true`, including the 5.6
    that refuses them in practice — which is why the exclusion is
    hand-maintained. A new generation is treated as defective until proven
    otherwise, since an excluded id can still be typed by hand but a broken one
    offered in the picker fails at agent runtime.
    """
    monkeypatch.setenv("OPENRAG_RUN_MODE", "oss")
    for provider in ("openai", "azure"):
        patterns = model_catalog.exclusions_for(provider)
        assert any(pattern.startswith("gpt-5.6-luna") for pattern in patterns), provider
        assert any(pattern.startswith("gpt-6-luna") for pattern in patterns), provider


@pytest.mark.asyncio
async def test_live_model_failure_does_not_block_other_provider_refreshes(monkeypatch):
    calls: list[str] = []

    async def fail(_credentials):
        calls.append("broken")
        raise ValueError("missing CA bundle")

    async def succeed(_credentials):
        calls.append("healthy")

    broken = SimpleNamespace(PROVIDER_KEY="broken", fetch_models=fail)
    healthy = SimpleNamespace(PROVIDER_KEY="healthy", fetch_models=succeed)
    providers = SimpleNamespace(stored_credentials=lambda _provider: {"api_key": "secret"})

    monkeypatch.setattr(model_catalog, "provider_enhancements", lambda: (broken, healthy))
    monkeypatch.setattr(
        model_catalog,
        "supported_provider_keys",
        lambda: frozenset({"broken", "healthy"}),
    )
    monkeypatch.setattr(
        "config.settings.get_openrag_config",
        lambda: SimpleNamespace(providers=providers),
    )

    await model_catalog.refresh_live_models()

    assert calls == ["broken", "healthy"]


@pytest.mark.asyncio
async def test_refresh_live_models_can_be_narrowed_to_one_provider(monkeypatch):
    """A caller asking about one provider must not wait on another's cluster."""
    calls: list[str] = []

    def lister(name):
        async def fetch(_credentials):
            calls.append(name)

        return fetch

    first = SimpleNamespace(PROVIDER_KEY="first", fetch_models=lister("first"))
    second = SimpleNamespace(PROVIDER_KEY="second", fetch_models=lister("second"))
    providers = SimpleNamespace(stored_credentials=lambda _provider: {"api_key": "secret"})

    monkeypatch.setattr(model_catalog, "provider_enhancements", lambda: (first, second))
    monkeypatch.setattr(
        model_catalog, "supported_provider_keys", lambda: frozenset({"first", "second"})
    )
    monkeypatch.setattr(
        "config.settings.get_openrag_config",
        lambda: SimpleNamespace(providers=providers),
    )

    await model_catalog.refresh_live_models(" Second ")

    assert calls == ["second"]