Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(litellm): derive reasoning capabilities from local metadata (#2240)

Selene29a5daaed3cca4
diff --git a/packages/core/src/repowise/core/providers/llm/litellm.py b/packages/core/src/repowise/core/providers/llm/litellm.py
index c3cdc9f23..89362922d 100644
--- a/packages/core/src/repowise/core/providers/llm/litellm.py
+++ b/packages/core/src/repowise/core/providers/llm/litellm.py
@@ -21,7 +21,7 @@ from __future__ import annotations
import contextlib
import os
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
from typing import TYPE_CHECKING, Any
import structlog
@@ -54,22 +54,164 @@ if TYPE_CHECKING:
log = structlog.get_logger(__name__)
-_LITELLM_REASONING_MODES: tuple[ReasoningMode, ...] = ("low", "medium", "high")
+_NON_TEXT_MODEL_MODES = {
+ "embedding",
+ "image_generation",
+ "image_edit",
+ "audio_transcription",
+ "audio_speech",
+ "rerank",
+ "video_generation",
+ "search",
+ "ocr",
+ "moderation",
+ "realtime",
+ "vector_store",
+}
+
+
+def _litellm_reasoning_modes_from_metadata(metadata: object) -> tuple[ReasoningMode, ...]:
+ """Map LiteLLM's model metadata to the exact portable effort choices."""
+
+ if not isinstance(metadata, dict):
+ return ()
+
+ explicit = metadata.get("reasoning_effort_levels")
+ effort_flags = (
+ "supports_none_reasoning_effort",
+ "supports_minimal_reasoning_effort",
+ "supports_low_reasoning_effort",
+ "supports_xhigh_reasoning_effort",
+ "supports_max_reasoning_effort",
+ )
+ if (
+ metadata.get("supports_reasoning") is not True
+ and not isinstance(explicit, list)
+ and not any(metadata.get(flag) is True for flag in effort_flags)
+ ):
+ return ()
+
+ if isinstance(explicit, list):
+ return tuple(
+ mode
+ for mode in ("none", "minimal", "low", "medium", "high", "xhigh", "max")
+ if mode in explicit and not (mode == "none" and metadata.get("thinking_always_on"))
+ )
+ modes: list[ReasoningMode] = []
+ if metadata.get("supports_none_reasoning_effort") is True and not metadata.get(
+ "thinking_always_on"
+ ):
+ modes.append("none")
+ if metadata.get("supports_minimal_reasoning_effort") is True:
+ modes.append("minimal")
+ if metadata.get("supports_low_reasoning_effort") is not False:
+ modes.append("low")
+ modes.extend(("medium", "high"))
+ if metadata.get("supports_xhigh_reasoning_effort") is True:
+ modes.append("xhigh")
+ if metadata.get("supports_max_reasoning_effort") is True:
+ modes.append("max")
+ return tuple(modes)
+
+
+def _litellm_metadata_decides_reasoning(metadata: object) -> bool:
+ if not isinstance(metadata, dict):
+ return False
+ if isinstance(metadata.get("reasoning_effort_levels"), list):
+ return True
+ fields = (
+ "supports_reasoning",
+ "supports_none_reasoning_effort",
+ "supports_minimal_reasoning_effort",
+ "supports_low_reasoning_effort",
+ "supports_xhigh_reasoning_effort",
+ "supports_max_reasoning_effort",
+ )
+ return any(metadata.get(field) is not None for field in fields)
+
+
+def _litellm_catalog_reasoning_modes(
+ model: str,
+ catalog: Mapping[str, object],
+ catalog_keys: Mapping[str, str],
+) -> tuple[ReasoningMode, ...] | None:
+ """Resolve direct or provider-prefixed metadata from the loaded catalog."""
+ if model not in catalog:
+ return None
+
+ metadata = catalog[model]
+ modes = _litellm_reasoning_modes_from_metadata(metadata)
+ if modes or _litellm_metadata_decides_reasoning(metadata):
+ return modes
+
+ if isinstance(metadata, dict):
+ provider = metadata.get("litellm_provider")
+ prefix = f"{provider}/" if isinstance(provider, str) else ""
+ if prefix and model.startswith(prefix):
+ bare_key = catalog_keys.get(model.removeprefix(prefix).casefold())
+ bare_metadata = catalog.get(bare_key) if bare_key is not None else None
+ bare_modes = _litellm_reasoning_modes_from_metadata(bare_metadata)
+ if bare_modes or _litellm_metadata_decides_reasoning(bare_metadata):
+ return bare_modes
-def _litellm_supports_reasoning(model: str) -> bool:
- try:
- import litellm # type: ignore[import-untyped]
+ return ()
+
+
+def _litellm_supported_reasoning_modes_from_sources(
+ litellm: Any,
+ model: str,
+ catalog: Mapping[str, object],
+ catalog_keys: Mapping[str, str],
+) -> tuple[ReasoningMode, ...]:
+ catalog_modes = _litellm_catalog_reasoning_modes(model, catalog, catalog_keys)
+ if catalog_modes is not None:
+ return catalog_modes
- return bool(litellm.supports_reasoning(model=model))
+ provider, separator, provider_model = model.partition("/")
+ if not separator:
+ return ()
+ try:
+ metadata: object = litellm.get_model_info(
+ provider_model,
+ custom_llm_provider=provider or None,
+ )
except Exception:
- return False
+ metadata = {}
+ modes = _litellm_reasoning_modes_from_metadata(metadata)
+ if modes:
+ return modes
+ if _litellm_metadata_decides_reasoning(metadata):
+ return ()
+ try:
+ if not bool(
+ litellm.supports_reasoning(
+ model=provider_model,
+ custom_llm_provider=provider or None,
+ )
+ ):
+ return ()
+ except Exception:
+ return ()
+
+ return _litellm_reasoning_modes_from_metadata({"supports_reasoning": True})
def _litellm_supported_reasoning_modes(model: str) -> tuple[ReasoningMode, ...]:
- if _litellm_supports_reasoning(model):
- return _LITELLM_REASONING_MODES
- return ()
+ try:
+ import litellm # type: ignore[import-untyped]
+
+ raw_catalog = getattr(litellm, "model_cost", {}) or {}
+ catalog = raw_catalog if isinstance(raw_catalog, Mapping) else {}
+ catalog_keys = {key.casefold(): key for key in catalog if isinstance(key, str)}
+ return _litellm_supported_reasoning_modes_from_sources(
+ litellm,
+ model,
+ catalog,
+ catalog_keys,
+ )
+ except Exception:
+ return ()
def _litellm_reasoning_kwargs(reasoning: ReasoningMode) -> dict[str, object]:
@@ -80,42 +222,54 @@ def _litellm_reasoning_kwargs(reasoning: ReasoningMode) -> dict[str, object]:
def _litellm_model_options(fallback_model: str) -> tuple[ProviderModelOption, ...]:
- fallback = fallback_model_option(
- fallback_model,
- reasoning_modes=("auto", *_litellm_supported_reasoning_modes(fallback_model)),
- )
try:
import litellm # type: ignore[import-untyped]
+ raw_catalog = getattr(litellm, "model_cost", {}) or {}
+ catalog = raw_catalog if isinstance(raw_catalog, Mapping) else {}
+ catalog_keys = {key.casefold(): key for key in catalog if isinstance(key, str)}
model_ids = sorted(
{
model
for model in getattr(litellm, "model_list", []) or []
- if isinstance(model, str) and model
+ if isinstance(model, str)
+ and model
+ and (
+ not isinstance(catalog.get(model), dict)
+ or catalog[model].get("mode") not in _NON_TEXT_MODEL_MODES
+ )
}
)
except Exception:
- return (fallback,)
+ return (fallback_model_option(fallback_model),)
+
+ fallback = fallback_model_option(
+ fallback_model,
+ reasoning_modes=(
+ "auto",
+ *_litellm_supported_reasoning_modes_from_sources(
+ litellm,
+ fallback_model,
+ catalog,
+ catalog_keys,
+ ),
+ ),
+ )
if not model_ids:
return (fallback,)
options: list[ProviderModelOption] = []
for model_id in model_ids:
- try:
- supports_reasoning = bool(litellm.supports_reasoning(model=model_id))
- except Exception:
- supports_reasoning = False
- reasoning_modes = (
- (
- "auto",
- *_LITELLM_REASONING_MODES,
- )
- if supports_reasoning
- else ("auto",)
+ model_modes = _litellm_supported_reasoning_modes_from_sources(
+ litellm,
+ model_id,
+ catalog,
+ catalog_keys,
)
+ reasoning_modes = ("auto", *model_modes)
notes = ""
- if supports_reasoning:
+ if model_modes:
notes = "LiteLLM reports reasoning support"
options.append(
ProviderModelOption(
diff --git a/tests/unit/test_providers/test_litellm_provider.py b/tests/unit/test_providers/test_litellm_provider.py
index fc5ef0ea4..341a829fe 100644
--- a/tests/unit/test_providers/test_litellm_provider.py
+++ b/tests/unit/test_providers/test_litellm_provider.py
@@ -20,8 +20,17 @@ def test_available_model_options_uses_litellm_model_list(monkeypatch):
)
monkeypatch.setattr(
litellm,
- "supports_reasoning",
- lambda *, model: model == "vendor/reasoner",
+ "model_cost",
+ {
+ "vendor/plain": {"supports_reasoning": False},
+ "vendor/reasoner": {"supports_reasoning": True},
+ },
+ raising=False,
+ )
+ monkeypatch.setattr(
+ litellm,
+ "get_model_info",
+ lambda _model: pytest.fail("catalog enumeration must stay local"),
raising=False,
)
@@ -35,6 +44,106 @@ def test_available_model_options_uses_litellm_model_list(monkeypatch):
assert "reasoning support" in reasoner.notes
+def test_discovery_falls_back_when_model_is_missing_from_catalog(monkeypatch):
+ litellm = pytest.importorskip("litellm", reason="litellm SDK not installed")
+ monkeypatch.setattr(litellm, "model_list", ["vendor/reasoner"], raising=False)
+ monkeypatch.setattr(litellm, "model_cost", {}, raising=False)
+ monkeypatch.setattr(
+ litellm,
+ "get_model_info",
+ lambda _model, **_kwargs: {"supports_reasoning": True},
+ raising=False,
+ )
+ monkeypatch.setattr(
+ litellm,
+ "supports_reasoning",
+ lambda **_kwargs: pytest.fail("model metadata must decide the capability"),
+ raising=False,
+ )
+ provider = LiteLLMProvider(model="vendor/reasoner")
+
+ option = provider.available_model_options()[0]
+
+ assert option.reasoning_modes == provider.supported_reasoning_modes()
+ assert option.reasoning_modes == ("auto", "low", "medium", "high")
+
+
+def test_discovery_uses_bare_model_metadata_for_indecisive_provider_row(monkeypatch):
+ litellm = pytest.importorskip("litellm", reason="litellm SDK not installed")
+ monkeypatch.setattr(litellm, "model_list", ["vendor/reasoner"], raising=False)
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ "vendor/reasoner": {"litellm_provider": "vendor", "mode": "chat"},
+ "Reasoner": {"supports_reasoning": True},
+ },
+ raising=False,
+ )
+ monkeypatch.setattr(
+ litellm,
+ "get_model_info",
+ lambda _model, **_kwargs: pytest.fail("the loaded catalog contains the fallback metadata"),
+ raising=False,
+ )
+ provider = LiteLLMProvider(model="vendor/reasoner")
+
+ option = provider.available_model_options()[0]
+
+ assert option.reasoning_modes == provider.supported_reasoning_modes()
+ assert option.reasoning_modes == ("auto", "low", "medium", "high")
+
+
+def test_available_model_options_uses_exact_litellm_effort_metadata(monkeypatch):
+ litellm = pytest.importorskip("litellm", reason="litellm SDK not installed")
+ monkeypatch.setattr(litellm, "model_list", ["vendor/reasoner"], raising=False)
+ metadata = {
+ "supports_reasoning": True,
+ "supports_none_reasoning_effort": True,
+ "supports_minimal_reasoning_effort": True,
+ "supports_low_reasoning_effort": False,
+ "supports_xhigh_reasoning_effort": True,
+ "supports_max_reasoning_effort": True,
+ }
+ monkeypatch.setattr(litellm, "model_cost", {"vendor/reasoner": metadata}, raising=False)
+ monkeypatch.setattr(
+ litellm,
+ "get_model_info",
+ lambda _model: pytest.fail("catalog enumeration must stay local"),
+ raising=False,
+ )
+
+ option = LiteLLMProvider(model="vendor/reasoner").available_model_options()[0]
+
+ assert option.reasoning_modes == (
+ "auto",
+ "none",
+ "minimal",
+ "medium",
+ "high",
+ "xhigh",
+ "max",
+ )
+
+
+def test_explicit_empty_effort_metadata_is_authoritative(monkeypatch):
+ litellm = pytest.importorskip("litellm", reason="litellm SDK not installed")
+ monkeypatch.setattr(
+ litellm,
+ "get_model_info",
+ lambda _model, **_kwargs: {"reasoning_effort_levels": []},
+ raising=False,
+ )
+ monkeypatch.setattr(
+ litellm,
+ "supports_reasoning",
+ lambda **_kwargs: pytest.fail("explicit metadata must win"),
+ raising=False,
+ )
+
+ assert LiteLLMProvider(model="vendor/plain").supported_reasoning_modes() == ("auto",)
+
+
async def test_generate_rejects_explicit_reasoning_before_litellm_call(monkeypatch):
litellm = pytest.importorskip("litellm", reason="litellm SDK not installed")
monkeypatch.setattr(litellm, "acompletion", AsyncMock(), raising=False)
@@ -71,7 +180,7 @@ async def test_generate_forwards_reasoning_effort(monkeypatch):
monkeypatch.setattr(
litellm,
"supports_reasoning",
- lambda *, model: model == "vendor/reasoner",
+ lambda *, model, **_kwargs: model == "reasoner",
raising=False,
)
@@ -81,3 +190,45 @@ async def test_generate_forwards_reasoning_effort(monkeypatch):
assert completion.call_args.kwargs["reasoning_effort"] == "high"
assert result.stop_reason == "max_tokens"
assert result.provider_stop_reason == "length"
+
+
+async def test_discovery_and_execution_accept_the_same_effort_flags(monkeypatch):
+ litellm = pytest.importorskip("litellm", reason="litellm SDK not installed")
+ metadata = {
+ "supports_reasoning": None,
+ "supports_none_reasoning_effort": False,
+ "supports_minimal_reasoning_effort": True,
+ "supports_low_reasoning_effort": None,
+ "supports_xhigh_reasoning_effort": False,
+ }
+ fake_response = type(
+ "Response",
+ (),
+ {
+ "choices": [
+ type(
+ "Choice",
+ (),
+ {
+ "message": type("Message", (), {"content": "ok"})(),
+ "finish_reason": "stop",
+ },
+ )()
+ ],
+ "usage": None,
+ },
+ )()
+ completion = AsyncMock(return_value=fake_response)
+ monkeypatch.setattr(litellm, "model_list", ["gpt-5-search-api"], raising=False)
+ monkeypatch.setattr(litellm, "model_cost", {"gpt-5-search-api": metadata}, raising=False)
+ monkeypatch.setattr(litellm, "get_model_info", lambda _model: metadata, raising=False)
+ monkeypatch.setattr(litellm, "supports_reasoning", lambda **_kwargs: False, raising=False)
+ monkeypatch.setattr(litellm, "acompletion", completion, raising=False)
+ provider = LiteLLMProvider(model="gpt-5-search-api")
+
+ option = provider.available_model_options()[0]
+ assert provider.supported_reasoning_modes() == option.reasoning_modes
+
+ await provider.generate("sys", "user", reasoning="minimal")
+
+ assert completion.call_args.kwargs["reasoning_effort"] == "minimal"