Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat: add LiteLLM as an AI Gateway (#395)

Prodman Devokadev356c6ab72b02
diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml
index 3207470..c16d8f9 100644
--- a/.github/workflows/build-reusable.yml
+++ b/.github/workflows/build-reusable.yml
@@ -310,7 +310,9 @@ jobs:
fi
retry 5 uv pip install --python "$BUILD_PY" scikit-build-core numpy swig Cython pybind11
if [[ "$RUNNER_OS" == "Linux" ]]; then
- retry 5 uv pip install --python "$BUILD_PY" auditwheel
+ # Install patchelf from PyPI (>=0.14.5) into the build venv; the
+ # ubuntu-22.04 apt patchelf is 0.14.3, which newer auditwheel rejects.
+ retry 5 uv pip install --python "$BUILD_PY" auditwheel "patchelf>=0.14.5"
elif [[ "$RUNNER_OS" == "macOS" ]]; then
retry 5 uv pip install --python "$BUILD_PY" delocate
else
diff --git a/apps/base_rag_example.py b/apps/base_rag_example.py
index 1aab9f4..0618ed2 100644
--- a/apps/base_rag_example.py
+++ b/apps/base_rag_example.py
@@ -32,6 +32,8 @@ try:
from leann.settings import (
resolve_atlascloud_api_key,
resolve_atlascloud_base_url,
+ resolve_litellm_api_key,
+ resolve_litellm_base_url,
resolve_ollama_host,
resolve_openai_api_key,
resolve_openai_base_url,
@@ -40,6 +42,12 @@ except ImportError:
# Minimal fallbacks if settings helpers are unavailable
import os
+ def resolve_litellm_api_key(value: str | None) -> str | None:
+ return value or os.getenv("LITELLM_API_KEY") or os.getenv("LEANN_LITELLM_API_KEY")
+
+ def resolve_litellm_base_url(value: str | None) -> str | None:
+ return value or os.getenv("LITELLM_BASE_URL") or os.getenv("LITELLM_API_BASE")
+
def resolve_ollama_host(value: str | None) -> str | None:
return value or os.getenv("LEANN_OLLAMA_HOST") or os.getenv("OLLAMA_HOST")
@@ -152,8 +160,17 @@ class BaseRAGExample(ABC):
"--llm",
type=str,
default="openai",
- choices=["openai", "ollama", "hf", "simulated", "atlascloud", "atlas-cloud", "atlas"],
- help="LLM backend: openai, ollama, hf, atlascloud, or simulated (default: openai)",
+ choices=[
+ "openai",
+ "ollama",
+ "hf",
+ "litellm",
+ "simulated",
+ "atlascloud",
+ "atlas-cloud",
+ "atlas",
+ ],
+ help="LLM backend: openai, ollama, hf, litellm, atlascloud, or simulated (default: openai)",
)
llm_group.add_argument(
"--llm-model",
@@ -288,6 +305,14 @@ class BaseRAGExample(ABC):
resolved_key = resolve_openai_api_key(args.llm_api_key)
if resolved_key:
config["api_key"] = resolved_key
+ elif args.llm == "litellm":
+ config["model"] = args.llm_model or "gpt-4o"
+ base_url = resolve_litellm_base_url(args.llm_api_base)
+ if base_url:
+ config["base_url"] = base_url
+ resolved_key = resolve_litellm_api_key(args.llm_api_key)
+ if resolved_key:
+ config["api_key"] = resolved_key
elif args.llm in {"atlascloud", "atlas-cloud", "atlas"}:
config["type"] = "atlascloud"
config["model"] = args.llm_model or "deepseek-ai/deepseek-v4-pro"
diff --git a/docs/configuration-guide.md b/docs/configuration-guide.md
index 66817d1..06fc1fa 100644
--- a/docs/configuration-guide.md
+++ b/docs/configuration-guide.md
@@ -316,6 +316,22 @@ leann build my-docs \
- **Cons**: More complex initial setup
- **Models**: `Qwen/Qwen3-1.7B-FP8`
+**LiteLLM** (`--llm litellm`)
+- **Pros**: One provider for 100+ backends (OpenAI, Anthropic, Gemini, Bedrock, Vertex, Azure, Groq, OpenRouter, ...) via a single interface; the model prefix picks the backend
+- **Cons**: Requires the optional dependency (`pip install "leann-core[litellm]"`)
+- **Models**: prefix the model with its provider, e.g. `gpt-4o` (OpenAI), `anthropic/claude-haiku-4-5`, `gemini/gemini-2.5-flash`, `openrouter/meta-llama/llama-3.1-70b-instruct`
+- **Credentials**: reads each provider's own env var automatically (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, ...). To route through a self-hosted LiteLLM proxy instead, pass `--api-base http://localhost:4000 --api-key <virtual-key>` (or set `LITELLM_BASE_URL` / `LITELLM_API_KEY`)
+- **Thinking Budget**: `--thinking-budget low/medium/high` maps to LiteLLM's unified `reasoning_effort` and is dropped for models that don't reason
+
+```bash
+# Ask via LiteLLM, routing to Anthropic (reads ANTHROPIC_API_KEY):
+leann ask my-docs --llm litellm --llm-model anthropic/claude-haiku-4-5
+
+# Or point at a self-hosted LiteLLM proxy:
+leann ask my-docs --llm litellm --llm-model gpt-4o \
+ --api-base http://localhost:4000 --api-key sk-my-virtual-key
+```
+
## Parameter Tuning Guide
### Search Complexity Parameters
diff --git a/packages/leann-core/pyproject.toml b/packages/leann-core/pyproject.toml
index 1b45f1c..c9040e0 100644
--- a/packages/leann-core/pyproject.toml
+++ b/packages/leann-core/pyproject.toml
@@ -44,6 +44,12 @@ dependencies = [
]
[project.optional-dependencies]
+litellm = [
+ # Unified gateway to 100+ LLM providers. Floor at 1.85 (recent stable with
+ # supports_reasoning/drop_params); capped below 2.0 to avoid a future
+ # major-version break.
+ "litellm>=1.85.0,<2.0",
+]
cpu = [
"torch==2.2.2; platform_system == 'Linux' and python_version < '3.13'",
]
diff --git a/packages/leann-core/src/leann/chat.py b/packages/leann-core/src/leann/chat.py
index 6f16eaf..f55d2fc 100644
--- a/packages/leann-core/src/leann/chat.py
+++ b/packages/leann-core/src/leann/chat.py
@@ -15,6 +15,8 @@ from .settings import (
resolve_anthropic_base_url,
resolve_atlascloud_api_key,
resolve_atlascloud_base_url,
+ resolve_litellm_api_key,
+ resolve_litellm_base_url,
resolve_minimax_api_key,
resolve_minimax_base_url,
resolve_novita_api_key,
@@ -1143,6 +1145,111 @@ class AtlasCloudChat(LLMInterface):
return f"Error: Could not get a response from Atlas Cloud. Details: {e}"
+class LiteLLMChat(LLMInterface):
+ """LLM interface for 100+ providers through LiteLLM's unified gateway.
+
+ LiteLLM routes to the right provider based on the model string's prefix,
+ so a single interface covers OpenAI, Anthropic, Gemini, Bedrock, Vertex,
+ Azure, Groq, OpenRouter, and many more:
+ - ``gpt-4o`` -> OpenAI
+ - ``anthropic/claude-haiku-4-5`` -> Anthropic
+ - ``gemini/gemini-2.5-flash`` -> Google Gemini
+ - ``openrouter/meta-llama/llama-3.1-70b-instruct`` -> OpenRouter
+ - ``bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0`` -> AWS Bedrock
+
+ Credentials are read from each provider's own environment variable
+ (``OPENAI_API_KEY``, ``ANTHROPIC_API_KEY``, ...) automatically. Set
+ ``base_url`` + ``api_key`` only to route through a self-hosted LiteLLM
+ proxy instead of calling providers directly.
+ """
+
+ def __init__(
+ self,
+ model: str = "gpt-4o",
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ ):
+ self.model = model
+ self.api_key = resolve_litellm_api_key(api_key)
+ self.base_url = resolve_litellm_base_url(base_url)
+
+ logger.info(
+ "Initializing LiteLLM Chat with model='%s' and base_url='%s'",
+ model,
+ self.base_url,
+ )
+
+ try:
+ import litellm # noqa: F401
+ except ImportError:
+ raise ImportError(
+ "The 'litellm' library is required for LiteLLM models. "
+ "Install it with 'pip install litellm' or 'pip install leann-core[litellm]'."
+ )
+
+ def ask(self, prompt: str, **kwargs) -> str:
+ import litellm
+
+ params: dict[str, Any] = {
+ "model": self.model,
+ "messages": [{"role": "user", "content": prompt}],
+ "temperature": kwargs.get("temperature", 0.7),
+ "max_tokens": kwargs.get("max_tokens", 1000),
+ # Silently drop kwargs a given provider doesn't accept so the same
+ # config works across OpenAI, Anthropic, Gemini, Bedrock, etc.
+ "drop_params": True,
+ }
+
+ # Only forward credentials/endpoint when explicitly set; otherwise let
+ # LiteLLM fall back to the provider's own environment variables.
+ if self.api_key:
+ params["api_key"] = self.api_key
+ if self.base_url:
+ params["api_base"] = self.base_url
+
+ # Map the repo's thinking budget onto LiteLLM's unified reasoning_effort
+ # param, but only for models that actually reason. Attaching it blindly
+ # errors on non-reasoning models when routed through a LiteLLM proxy
+ # (client-side drop_params isn't honored server-side).
+ thinking_budget = kwargs.get("thinking_budget")
+ if thinking_budget in ("low", "medium", "high"):
+ try:
+ supports_reasoning = litellm.supports_reasoning(self.model)
+ except Exception:
+ supports_reasoning = False
+ if supports_reasoning:
+ params["reasoning_effort"] = thinking_budget
+ else:
+ logger.warning(
+ "Thinking budget '%s' requested but model '%s' does not support "
+ "reasoning; ignoring it.",
+ thinking_budget,
+ self.model,
+ )
+
+ if "top_p" in kwargs:
+ params["top_p"] = kwargs["top_p"]
+
+ # Forward any remaining explicit kwargs (already-handled keys excluded).
+ for k, v in kwargs.items():
+ if k not in {"temperature", "max_tokens", "top_p", "thinking_budget"}:
+ params[k] = v
+
+ logger.info(f"Sending request to LiteLLM with model {self.model}")
+
+ try:
+ response = cast(Any, litellm.completion(**params))
+ print(
+ f"Total tokens = {response.usage.total_tokens}, prompt tokens = {response.usage.prompt_tokens}, completion tokens = {response.usage.completion_tokens}"
+ )
+ if response.choices[0].finish_reason == "length":
+ print("The query is exceeding the maximum allowed number of tokens")
+ return response.choices[0].message.content.strip()
+ except Exception as e:
+ logger.error(f"Error communicating with LiteLLM: {e}")
+ return f"Error: Could not get a response from LiteLLM. Details: {e}"
+
+
class SimulatedChat(LLMInterface):
"""A simple simulated chat for testing and development."""
@@ -1219,6 +1326,12 @@ def get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:
api_key=llm_config.get("api_key"),
base_url=llm_config.get("base_url"),
)
+ elif llm_type == "litellm":
+ return LiteLLMChat(
+ model=model or "gpt-4o",
+ api_key=llm_config.get("api_key"),
+ base_url=llm_config.get("base_url"),
+ )
elif llm_type == "simulated":
return SimulatedChat()
else:
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index 4cacdee..1511e66 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -24,6 +24,8 @@ from .settings import (
resolve_anthropic_base_url,
resolve_atlascloud_api_key,
resolve_atlascloud_base_url,
+ resolve_litellm_api_key,
+ resolve_litellm_base_url,
resolve_minimax_api_key,
resolve_minimax_base_url,
resolve_ollama_host,
@@ -656,6 +658,7 @@ Examples:
"hf",
"openai",
"anthropic",
+ "litellm",
"minimax",
"novita",
"atlascloud",
@@ -741,6 +744,7 @@ Examples:
"hf",
"openai",
"anthropic",
+ "litellm",
"minimax",
"novita",
"atlascloud",
@@ -3421,6 +3425,15 @@ Examples:
llm_config["base_url"] = resolve_anthropic_base_url(args.api_base)
if args.api_key:
llm_config["api_key"] = args.api_key
+ elif args.llm == "litellm":
+ # LiteLLM infers the provider from the model prefix and reads that
+ # provider's own env var; base_url/api_key are only for a proxy.
+ base_url = resolve_litellm_base_url(args.api_base)
+ if base_url:
+ llm_config["base_url"] = base_url
+ resolved_api_key = resolve_litellm_api_key(args.api_key)
+ if resolved_api_key:
+ llm_config["api_key"] = resolved_api_key
elif args.llm == "minimax":
llm_config["base_url"] = resolve_minimax_base_url(args.api_base)
resolved_api_key = resolve_minimax_api_key(args.api_key)
@@ -3716,6 +3729,15 @@ Examples:
llm_config["base_url"] = resolve_anthropic_base_url(args.api_base)
if args.api_key:
llm_config["api_key"] = args.api_key
+ elif args.llm == "litellm":
+ # LiteLLM infers the provider from the model prefix and reads that
+ # provider's own env var; base_url/api_key are only for a proxy.
+ base_url = resolve_litellm_base_url(args.api_base)
+ if base_url:
+ llm_config["base_url"] = base_url
+ resolved_api_key = resolve_litellm_api_key(args.api_key)
+ if resolved_api_key:
+ llm_config["api_key"] = resolved_api_key
elif args.llm == "minimax":
llm_config["base_url"] = resolve_minimax_base_url(args.api_base)
resolved_api_key = resolve_minimax_api_key(args.api_key)
diff --git a/packages/leann-core/src/leann/settings.py b/packages/leann-core/src/leann/settings.py
index a43e217..491c0c8 100644
--- a/packages/leann-core/src/leann/settings.py
+++ b/packages/leann-core/src/leann/settings.py
@@ -94,6 +94,43 @@ def resolve_anthropic_api_key(explicit: str | None = None) -> str | None:
return os.getenv("ANTHROPIC_API_KEY")
+def resolve_litellm_base_url(explicit: str | None = None) -> str | None:
+ """Resolve an optional base URL for LiteLLM.
+
+ Unlike the single-provider resolvers there is no default: when unset,
+ LiteLLM talks directly to the provider inferred from the model string
+ (e.g. ``anthropic/claude-...``). Set a base URL only to route through a
+ self-hosted LiteLLM proxy.
+ """
+
+ candidates = (
+ explicit,
+ os.getenv("LEANN_LITELLM_BASE_URL"),
+ os.getenv("LITELLM_BASE_URL"),
+ os.getenv("LITELLM_API_BASE"),
+ )
+
+ for candidate in candidates:
+ if candidate:
+ return _clean_url(candidate)
+
+ return None
+
+
+def resolve_litellm_api_key(explicit: str | None = None) -> str | None:
+ """Resolve an optional API key for LiteLLM.
+
+ When unset, LiteLLM reads the underlying provider's own environment
+ variable (``OPENAI_API_KEY``, ``ANTHROPIC_API_KEY``, ...). Set this only
+ when talking to a LiteLLM proxy that expects a virtual key.
+ """
+
+ if explicit:
+ return explicit
+
+ return os.getenv("LITELLM_API_KEY") or os.getenv("LEANN_LITELLM_API_KEY")
+
+
def resolve_minimax_base_url(explicit: str | None = None) -> str:
"""Resolve the base URL for MiniMax-compatible services."""
diff --git a/tests/test_litellm_provider.py b/tests/test_litellm_provider.py
new file mode 100644
index 0000000..7fd58f6
--- /dev/null
+++ b/tests/test_litellm_provider.py
@@ -0,0 +1,269 @@
+"""
+Tests for LiteLLM provider integration.
+
+These tests validate LiteLLM provider settings, chat class, and factory
+integration. We import from leann.settings and leann.chat directly to avoid
+triggering the full leann.__init__ import chain which requires C++ backend
+builds. A lightweight fake ``litellm`` module is installed so the provider
+imports cleanly even when the optional dependency isn't present.
+"""
+
+import os
+import sys
+import types
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+# Add the leann-core source to sys.path so we can import submodules
+# without triggering __init__.py's backend imports.
+_LEANN_SRC = os.path.join(os.path.dirname(__file__), "..", "packages", "leann-core", "src")
+if _LEANN_SRC not in sys.path:
+ sys.path.insert(0, os.path.abspath(_LEANN_SRC))
+
+# Prevent leann.__init__ from running its heavy imports by pre-registering
+# a lightweight stub in sys.modules (if not already present).
+if "leann" not in sys.modules:
+ _stub = types.ModuleType("leann")
+ _stub.__path__ = [os.path.join(os.path.abspath(_LEANN_SRC), "leann")]
+ sys.modules["leann"] = _stub
+
+# Install a fake `litellm` module so `import litellm` succeeds even when the
+# optional dependency isn't installed. Real installs are left untouched.
+if "litellm" not in sys.modules:
+ _fake_litellm = types.ModuleType("litellm")
+ # Assign through __dict__ so static type checkers don't flag attribute
+ # writes on a bare ModuleType (the attributes are what `import litellm`
+ # then resolves).
+ _fake_litellm.__dict__["completion"] = MagicMock(name="litellm.completion")
+ _fake_litellm.__dict__["supports_reasoning"] = MagicMock(name="litellm.supports_reasoning")
+ sys.modules["litellm"] = _fake_litellm
+
+# Now we can safely import the modules we actually test.
+from leann.settings import ( # noqa: E402
+ resolve_litellm_api_key,
+ resolve_litellm_base_url,
+)
+
+
+def _make_response(content: str, finish_reason: str = "stop"):
+ """Build an OpenAI-shaped response mock like litellm.completion returns."""
+ response = MagicMock()
+ response.choices = [MagicMock()]
+ response.choices[0].message.content = content
+ response.choices[0].finish_reason = finish_reason
+ response.usage.total_tokens = 100
+ response.usage.prompt_tokens = 50
+ response.usage.completion_tokens = 50
+ return response
+
+
+class TestLiteLLMSettings:
+ """Test LiteLLM settings resolver functions."""
+
+ def test_resolve_api_key_explicit(self):
+ assert resolve_litellm_api_key("test-key") == "test-key"
+
+ def test_resolve_api_key_from_litellm_env(self):
+ with patch.dict(os.environ, {"LITELLM_API_KEY": "lite-key"}, clear=True):
+ assert resolve_litellm_api_key() == "lite-key"
+
+ def test_resolve_api_key_from_leann_env(self):
+ with patch.dict(os.environ, {"LEANN_LITELLM_API_KEY": "leann-key"}, clear=True):
+ assert resolve_litellm_api_key() == "leann-key"
+
+ def test_resolve_api_key_none(self):
+ with patch.dict(os.environ, {}, clear=True):
+ assert resolve_litellm_api_key() is None
+
+ def test_resolve_base_url_default_none(self):
+ # Unlike single-provider resolvers, LiteLLM has no default base URL:
+ # it routes directly to the provider inferred from the model string.
+ with patch.dict(os.environ, {}, clear=True):
+ assert resolve_litellm_base_url() is None
+
+ def test_resolve_base_url_explicit(self):
+ assert resolve_litellm_base_url("https://proxy.url/v1") == "https://proxy.url/v1"
+
+ def test_resolve_base_url_from_litellm_env(self):
+ with patch.dict(os.environ, {"LITELLM_BASE_URL": "https://env.url/v1"}, clear=True):
+ assert resolve_litellm_base_url() == "https://env.url/v1"
+
+ def test_resolve_base_url_from_leann_env(self):
+ with patch.dict(os.environ, {"LEANN_LITELLM_BASE_URL": "https://leann.url/v1"}, clear=True):
+ assert resolve_litellm_base_url() == "https://leann.url/v1"
+
+ def test_resolve_base_url_from_api_base_env(self):
+ with patch.dict(os.environ, {"LITELLM_API_BASE": "https://apibase.url/v1"}, clear=True):
+ assert resolve_litellm_base_url() == "https://apibase.url/v1"
+
+ def test_resolve_base_url_strips_trailing_slash(self):
+ assert resolve_litellm_base_url("https://proxy.url/v1/") == "https://proxy.url/v1"
+
+
+class TestLiteLLMChat:
+ """Test LiteLLMChat class."""
+
+ def test_init_does_not_require_api_key(self):
+ # LiteLLM reads each provider's own env var, so no key is needed at init.
+ from leann.chat import LiteLLMChat
+
+ with patch.dict(os.environ, {}, clear=True):
+ chat = LiteLLMChat()
+ assert chat.model == "gpt-4o"
+ assert chat.api_key is None
+ assert chat.base_url is None
+
+ def test_init_custom_model(self):
+ from leann.chat import LiteLLMChat
+
+ chat = LiteLLMChat(model="anthropic/claude-haiku-4-5")
+ assert chat.model == "anthropic/claude-haiku-4-5"
+
+ def test_init_proxy_config(self):
+ from leann.chat import LiteLLMChat
+
+ chat = LiteLLMChat(api_key="proxy-key", base_url="https://proxy.url/v1")
+ assert chat.api_key == "proxy-key"
+ assert chat.base_url == "https://proxy.url/v1"
+
+ def test_ask_returns_response(self):
+ from leann.chat import LiteLLMChat
+
+ with patch("litellm.completion", return_value=_make_response("Hello from LiteLLM!")) as m:
+ chat = LiteLLMChat(model="gpt-4o")
+ result = chat.ask("Hello")
+
+ assert result == "Hello from LiteLLM!"
+ m.assert_called_once()
+
+ def test_ask_sets_drop_params_by_default(self):
+ from leann.chat import LiteLLMChat
+
+ with patch("litellm.completion", return_value=_make_response("ok")) as m:
+ LiteLLMChat(model="anthropic/claude-haiku-4-5").ask("Hello")
+
+ call_kwargs = m.call_args[1]
+ assert call_kwargs["drop_params"] is True
+ assert call_kwargs["model"] == "anthropic/claude-haiku-4-5"
+ assert call_kwargs["messages"] == [{"role": "user", "content": "Hello"}]
+
+ def test_ask_forwards_credentials_only_when_set(self):
+ from leann.chat import LiteLLMChat
+
+ # No proxy config -> api_key/api_base omitted so LiteLLM uses env vars.
+ with patch("litellm.completion", return_value=_make_response("ok")) as m:
+ with patch.dict(os.environ, {}, clear=True):
+ LiteLLMChat(model="gpt-4o").ask("Hello")
+ call_kwargs = m.call_args[1]
+ assert "api_key" not in call_kwargs
+ assert "api_base" not in call_kwargs
+
+ # Proxy config -> forwarded as api_key/api_base.
+ with patch("litellm.completion", return_value=_make_response("ok")) as m:
+ LiteLLMChat(api_key="proxy-key", base_url="https://proxy.url/v1").ask("Hello")
+ call_kwargs = m.call_args[1]
+ assert call_kwargs["api_key"] == "proxy-key"
+ assert call_kwargs["api_base"] == "https://proxy.url/v1"
+
+ def test_ask_with_kwargs(self):
+ from leann.chat import LiteLLMChat
+
+ with patch("litellm.completion", return_value=_make_response("Response")) as m:
+ LiteLLMChat(model="gpt-4o").ask("Hello", temperature=0.5, max_tokens=500, top_p=0.9)
+
+ call_kwargs = m.call_args[1]
+ assert call_kwargs["temperature"] == 0.5
+ assert call_kwargs["max_tokens"] == 500
+ assert call_kwargs["top_p"] == 0.9
+
+ def test_ask_maps_thinking_budget_for_reasoning_model(self):
+ from leann.chat import LiteLLMChat
+
+ with (
+ patch("litellm.completion", return_value=_make_response("Response")) as m,
+ patch("litellm.supports_reasoning", return_value=True),
+ ):
+ LiteLLMChat(model="anthropic/claude-haiku-4-5").ask("Hello", thinking_budget="high")
+
+ call_kwargs = m.call_args[1]
+ assert call_kwargs["reasoning_effort"] == "high"
+ # thinking_budget itself must not leak through to the API call.
+ assert "thinking_budget" not in call_kwargs
+
+ def test_ask_skips_thinking_budget_for_non_reasoning_model(self):
+ from leann.chat import LiteLLMChat
+
+ # Non-reasoning models must NOT get reasoning_effort: a LiteLLM proxy
+ # rejects the unsupported param (client-side drop_params isn't honored).
+ with (
+ patch("litellm.completion", return_value=_make_response("Response")) as m,
+ patch("litellm.supports_reasoning", return_value=False),
+ ):
+ LiteLLMChat(model="gpt-4.1-mini").ask("Hello", thinking_budget="high")
+
+ call_kwargs = m.call_args[1]
+ assert "reasoning_effort" not in call_kwargs
+ assert "thinking_budget" not in call_kwargs
+
+ def test_ask_handles_error(self):
+ from leann.chat import LiteLLMChat
+
+ with patch("litellm.completion", side_effect=Exception("API error")):
+ result = LiteLLMChat(model="gpt-4o").ask("Hello")
+
+ assert "Error" in result
+ assert "LiteLLM" in result
+
+
+class TestGetLLMFactory:
+ """Test get_llm factory function with litellm type."""
+
+ def test_get_llm_litellm(self):
+ from leann.chat import LiteLLMChat, get_llm
+
+ llm = get_llm({"type": "litellm"})
+ assert isinstance(llm, LiteLLMChat)
+ assert llm.model == "gpt-4o"
+
+ def test_get_llm_litellm_custom_model(self):
+ from leann.chat import LiteLLMChat, get_llm
+
+ llm = get_llm({"type": "litellm", "model": "gemini/gemini-2.5-flash"})
+ assert isinstance(llm, LiteLLMChat)
+ assert llm.model == "gemini/gemini-2.5-flash"
+
+ def test_get_llm_litellm_proxy(self):
+ from leann.chat import LiteLLMChat, get_llm
+
+ llm = get_llm(
+ {
+ "type": "litellm",
+ "api_key": "proxy-key",
+ "base_url": "https://proxy.url/v1",
+ }
+ )
+ assert isinstance(llm, LiteLLMChat)
+ assert llm.api_key == "proxy-key"
+ assert llm.base_url == "https://proxy.url/v1"
+
+
+ not os.getenv("LITELLM_LIVE_TEST"),
+ reason="LITELLM_LIVE_TEST not set; skipping live API test",
+)
+class TestLiteLLMLiveAPI:
+ """Live API tests for LiteLLM provider.
+
+ Requires a real provider key (e.g. OPENAI_API_KEY or ANTHROPIC_API_KEY)
+ and LITELLM_LIVE_TEST=1. Set --model via LITELLM_LIVE_MODEL (default gpt-4o).
+ """
+
+ def test_litellm_live(self):
+ from leann.chat import LiteLLMChat
+
+ model = os.getenv("LITELLM_LIVE_MODEL", "gpt-4o")
+ chat = LiteLLMChat(model=model)
+ response = chat.ask("What is 1+1? Reply with just the number.", max_tokens=10)
+ assert isinstance(response, str)
+ assert len(response) > 0