Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

feat: add Novita AI as LLM provider (#294)

Alex651f4c596168
diff --git a/packages/leann-core/src/leann/chat.py b/packages/leann-core/src/leann/chat.py
index 0206a0e..52426d1 100644
--- a/packages/leann-core/src/leann/chat.py
+++ b/packages/leann-core/src/leann/chat.py
@@ -17,6 +17,8 @@ from .settings import (
resolve_anthropic_base_url,
resolve_minimax_api_key,
resolve_minimax_base_url,
+ resolve_novita_api_key,
+ resolve_novita_base_url,
resolve_ollama_host,
resolve_openai_api_key,
resolve_openai_base_url,
@@ -1010,6 +1012,76 @@ class MiniMaxChat(LLMInterface):
return f"Error: Could not get a response from MiniMax. Details: {e}"
+class NovitaChat(LLMInterface):
+ """LLM interface for Novita AI models via the OpenAI-compatible API.
+
+ Supported models include:
+ - moonshotai/kimi-k2.5 (default): 262K context, MoE with function calling,
+ structured output, reasoning, and vision support.
+ - zai-org/glm-5: 202K context, MoE with function calling, structured output,
+ reasoning support.
+ - minimax/minimax-m2.5: 204K context, MoE with function calling, structured
+ output, reasoning support.
+
+ See CLAUDE.md for the full model catalog with pricing and features.
+ """
+
+ def __init__(
+ self,
+ model: str = "moonshotai/kimi-k2.5",
+ api_key: Optional[str] = None,
+ base_url: Optional[str] = None,
+ ):
+ self.model = model
+ self.base_url = resolve_novita_base_url(base_url)
+ self.api_key = resolve_novita_api_key(api_key)
+
+ if not self.api_key:
+ raise ValueError(
+ "Novita API key is required. Set NOVITA_API_KEY environment variable or pass api_key parameter."
+ )
+
+ logger.info(
+ "Initializing Novita Chat with model='%s' and base_url='%s'",
+ model,
+ self.base_url,
+ )
+
+ try:
+ import openai
+
+ self.client = openai.OpenAI(api_key=self.api_key, base_url=self.base_url)
+ except ImportError:
+ raise ImportError(
+ "The 'openai' library is required for Novita models. Please install it with 'pip install openai'."
+ )
+
+ def ask(self, prompt: str, **kwargs) -> str:
+ params = {
+ "model": self.model,
+ "messages": [{"role": "user", "content": prompt}],
+ "temperature": kwargs.get("temperature", 0.7),
+ "max_tokens": kwargs.get("max_tokens", 1000),
+ }
+
+ if "top_p" in kwargs:
+ params["top_p"] = kwargs["top_p"]
+
+ logger.info(f"Sending request to Novita with model {self.model}")
+
+ try:
+ response = cast(Any, self.client.chat.completions).create(**params)
+ logger.info(
+ f"Total tokens = {response.usage.total_tokens}, prompt tokens = {response.usage.prompt_tokens}, completion tokens = {response.usage.completion_tokens}"
+ )
+ if response.choices[0].finish_reason == "length":
+ logger.warning("The query is exceeding the maximum allowed number of tokens")
+ return response.choices[0].message.content.strip()
+ except Exception as e:
+ logger.error(f"Error communicating with Novita: {e}")
+ return f"Error: Could not get a response from Novita. Details: {e}"
+
+
class SimulatedChat(LLMInterface):
"""A simple simulated chat for testing and development."""
@@ -1074,6 +1146,12 @@ def get_llm(llm_config: Optional[dict[str, Any]] = None) -> LLMInterface:
api_key=llm_config.get("api_key"),
base_url=llm_config.get("base_url"),
)
+ elif llm_type == "novita":
+ return NovitaChat(
+ model=model or "moonshotai/kimi-k2.5",
+ api_key=llm_config.get("api_key"),
+ base_url=llm_config.get("base_url"),
+ )
elif llm_type == "simulated":
return SimulatedChat()
else:
diff --git a/packages/leann-core/src/leann/cli.py b/packages/leann-core/src/leann/cli.py
index 29852b3..fec66f0 100644
--- a/packages/leann-core/src/leann/cli.py
+++ b/packages/leann-core/src/leann/cli.py
@@ -499,7 +499,7 @@ Examples:
"--llm",
type=str,
default="ollama",
- choices=["simulated", "ollama", "hf", "openai", "anthropic", "minimax"],
+ choices=["simulated", "ollama", "hf", "openai", "anthropic", "minimax", "novita"],
help="LLM provider (default: ollama)",
)
ask_parser.add_argument(
@@ -562,7 +562,7 @@ Examples:
"--llm",
type=str,
default="ollama",
- choices=["simulated", "ollama", "hf", "openai", "anthropic", "minimax"],
+ choices=["simulated", "ollama", "hf", "openai", "anthropic", "minimax", "novita"],
help="LLM provider (default: ollama)",
)
react_parser.add_argument(
diff --git a/packages/leann-core/src/leann/settings.py b/packages/leann-core/src/leann/settings.py
index e3b8e9e..3d6afb8 100644
--- a/packages/leann-core/src/leann/settings.py
+++ b/packages/leann-core/src/leann/settings.py
@@ -3,14 +3,18 @@
from __future__ import annotations
import json
+import logging
import os
from typing import Any
+logger = logging.getLogger(__name__)
+
# Default fallbacks to preserve current behaviour while keeping them in one place.
_DEFAULT_OLLAMA_HOST = "http://localhost:11434"
_DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"
_DEFAULT_ANTHROPIC_BASE_URL = "https://api.anthropic.com"
_DEFAULT_MINIMAX_BASE_URL = "https://api.minimax.io/v1"
+_DEFAULT_NOVITA_BASE_URL = "https://api.novita.ai/openai"
def _clean_url(value: str) -> str:
@@ -114,6 +118,42 @@ def resolve_minimax_api_key(explicit: str | None = None) -> str | None:
return os.getenv("MINIMAX_API_KEY")
+def resolve_novita_base_url(explicit: str | None = None) -> str:
+ """Resolve the base URL for Novita AI services."""
+
+ candidates = (
+ explicit,
+ os.getenv("LEANN_NOVITA_BASE_URL"),
+ os.getenv("NOVITA_BASE_URL"),
+ os.getenv("OPENAI_BASE_URL"), # Fallback to OpenAI base URL
+ )
+
+ for candidate in candidates:
+ if candidate:
+ return _clean_url(candidate)
+
+ return _clean_url(_DEFAULT_NOVITA_BASE_URL)
+
+
+def resolve_novita_api_key(explicit: str | None = None) -> str | None:
+ """Resolve the API key for Novita AI services."""
+
+ if explicit:
+ return explicit
+
+ novita_key = os.getenv("NOVITA_API_KEY")
+ if novita_key:
+ return novita_key
+
+ openai_key = os.getenv("OPENAI_API_KEY")
+ if openai_key:
+ logger.warning(
+ "NOVITA_API_KEY not set, falling back to OPENAI_API_KEY. "
+ "This may cause authentication issues if the OpenAI key is not valid for Novita AI."
+ )
+ return openai_key
+
+
def encode_provider_options(options: dict[str, Any] | None) -> str | None:
"""Serialize provider options for child processes."""
diff --git a/tests/test_novita_provider.py b/tests/test_novita_provider.py
new file mode 100644
index 0000000..e7374de
--- /dev/null
+++ b/tests/test_novita_provider.py
@@ -0,0 +1,246 @@
+"""
+Tests for Novita AI provider integration.
+
+These tests validate Novita provider settings, chat class, and factory integration.
+We import from leann.settings and leann.chat directly to avoid triggering
+the full leann.__init__ import chain which requires C++ backend builds.
+"""
+
+import os
+import sys
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+# Add the leann-core source to sys.path so we can import submodules
+# without triggering __init__.py's backend imports.
+_LEANN_SRC = os.path.join(os.path.dirname(__file__), "..", "packages", "leann-core", "src")
+if _LEANN_SRC not in sys.path:
+ sys.path.insert(0, os.path.abspath(_LEANN_SRC))
+
+# Prevent leann.__init__ from running its heavy imports by pre-registering
+# a lightweight stub in sys.modules (if not already present).
+if "leann" not in sys.modules:
+ import types
+
+ _stub = types.ModuleType("leann")
+ _stub.__path__ = [os.path.join(os.path.abspath(_LEANN_SRC), "leann")]
+ sys.modules["leann"] = _stub
+
+# Now we can safely import the modules we actually test.
+from leann.settings import ( # noqa: E402
+ resolve_novita_api_key,
+ resolve_novita_base_url,
+)
+
+
+class TestNovitaSettings:
+ """Test Novita settings resolver functions."""
+
+ def test_resolve_novita_api_key_explicit(self):
+ assert resolve_novita_api_key("test-key") == "test-key"
+
+ def test_resolve_novita_api_key_from_novita_env(self):
+ with patch.dict(os.environ, {"NOVITA_API_KEY": "novita-key"}, clear=True):
+ assert resolve_novita_api_key() == "novita-key"
+
+ def test_resolve_novita_api_key_fallback_to_openai(self):
+ with patch.dict(os.environ, {"OPENAI_API_KEY": "openai-key"}, clear=True):
+ assert resolve_novita_api_key() == "openai-key"
+
+ def test_resolve_novita_api_key_novita_takes_precedence(self):
+ with patch.dict(
+ os.environ,
+ {"NOVITA_API_KEY": "novita-key", "OPENAI_API_KEY": "openai-key"},
+ ):
+ assert resolve_novita_api_key() == "novita-key"
+
+ def test_resolve_novita_api_key_none(self):
+ with patch.dict(os.environ, {}, clear=True):
+ assert resolve_novita_api_key() is None
+
+ def test_resolve_novita_base_url_default(self):
+ with patch.dict(os.environ, {}, clear=True):
+ assert resolve_novita_base_url() == "https://api.novita.ai/openai"
+
+ def test_resolve_novita_base_url_explicit(self):
+ assert resolve_novita_base_url("https://custom.url/v1") == "https://custom.url/v1"
+
+ def test_resolve_novita_base_url_from_novita_env(self):
+ with patch.dict(os.environ, {"NOVITA_BASE_URL": "https://env.url/v1"}):
+ assert resolve_novita_base_url() == "https://env.url/v1"
+
+ def test_resolve_novita_base_url_from_leann_env(self):
+ with patch.dict(os.environ, {"LEANN_NOVITA_BASE_URL": "https://leann.url/v1"}):
+ assert resolve_novita_base_url() == "https://leann.url/v1"
+
+ def test_resolve_novita_base_url_fallback_to_openai_env(self):
+ with patch.dict(os.environ, {"OPENAI_BASE_URL": "https://openai.url/v1"}, clear=True):
+ assert resolve_novita_base_url() == "https://openai.url/v1"
+
+ def test_resolve_novita_base_url_strips_trailing_slash(self):
+ assert (
+ resolve_novita_base_url("https://api.novita.ai/openai/")
+ == "https://api.novita.ai/openai"
+ )
+
+
+class TestNovitaChat:
+ """Test NovitaChat class."""
+
+ def test_init_requires_api_key(self):
+ from leann.chat import NovitaChat
+
+ with patch.dict(os.environ, {}, clear=True):
+ with pytest.raises(ValueError, match="Novita API key is required"):
+ NovitaChat(api_key=None)
+
+ @patch("openai.OpenAI")
+ def test_init_with_api_key(self, mock_openai_cls):
+ from leann.chat import NovitaChat
+
+ chat = NovitaChat(api_key="test-key")
+ assert chat.model == "moonshotai/kimi-k2.5"
+ assert chat.api_key == "test-key"
+ assert chat.base_url == "https://api.novita.ai/openai"
+ mock_openai_cls.assert_called_once_with(
+ api_key="test-key", base_url="https://api.novita.ai/openai"
+ )
+
+ @patch("openai.OpenAI")
+ def test_init_custom_model(self, mock_openai_cls):
+ from leann.chat import NovitaChat
+
+ chat = NovitaChat(model="zai-org/glm-5", api_key="test-key")
+ assert chat.model == "zai-org/glm-5"
+
+ @patch("openai.OpenAI")
+ def test_init_custom_base_url(self, mock_openai_cls):
+ from leann.chat import NovitaChat
+
+ chat = NovitaChat(api_key="test-key", base_url="https://custom.novita.url/v1")
+ assert chat.base_url == "https://custom.novita.url/v1"
+
+ @patch("openai.OpenAI")
+ def test_ask_returns_response(self, mock_openai_cls):
+ from leann.chat import NovitaChat
+
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+
+ mock_response = MagicMock()
+ mock_response.choices = [MagicMock()]
+ mock_response.choices[0].message.content = "Hello from Novita!"
+ mock_response.choices[0].finish_reason = "stop"
+ mock_response.usage.total_tokens = 100
+ mock_response.usage.prompt_tokens = 50
+ mock_response.usage.completion_tokens = 50
+ mock_client.chat.completions.create.return_value = mock_response
+
+ chat = NovitaChat(api_key="test-key")
+ result = chat.ask("Hello")
+
+ assert result == "Hello from Novita!"
+ mock_client.chat.completions.create.assert_called_once()
+
+ @patch("openai.OpenAI")
+ def test_ask_with_kwargs(self, mock_openai_cls):
+ from leann.chat import NovitaChat
+
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+
+ mock_response = MagicMock()
+ mock_response.choices = [MagicMock()]
+ mock_response.choices[0].message.content = "Response"
+ mock_response.choices[0].finish_reason = "stop"
+ mock_response.usage.total_tokens = 50
+ mock_response.usage.prompt_tokens = 25
+ mock_response.usage.completion_tokens = 25
+ mock_client.chat.completions.create.return_value = mock_response
+
+ chat = NovitaChat(api_key="test-key")
+ chat.ask("Hello", temperature=0.5, max_tokens=500, top_p=0.9)
+
+ call_kwargs = mock_client.chat.completions.create.call_args[1]
+ assert call_kwargs["temperature"] == 0.5
+ assert call_kwargs["max_tokens"] == 500
+ assert call_kwargs["top_p"] == 0.9
+
+ @patch("openai.OpenAI")
+ def test_ask_handles_error(self, mock_openai_cls):
+ from leann.chat import NovitaChat
+
+ mock_client = MagicMock()
+ mock_openai_cls.return_value = mock_client
+ mock_client.chat.completions.create.side_effect = Exception("API error")
+
+ chat = NovitaChat(api_key="test-key")
+ result = chat.ask("Hello")
+
+ assert "Error" in result
+ assert "Novita" in result
+
+
+class TestGetLLMFactory:
+ """Test get_llm factory function with novita type."""
+
+ @patch("openai.OpenAI")
+ def test_get_llm_novita(self, mock_openai_cls):
+ from leann.chat import NovitaChat, get_llm
+
+ llm = get_llm({"type": "novita", "api_key": "test-key"})
+ assert isinstance(llm, NovitaChat)
+ assert llm.model == "moonshotai/kimi-k2.5"
+
+ @patch("openai.OpenAI")
+ def test_get_llm_novita_custom_model(self, mock_openai_cls):
+ from leann.chat import NovitaChat, get_llm
+
+ llm = get_llm(
+ {
+ "type": "novita",
+ "model": "zai-org/glm-5",
+ "api_key": "test-key",
+ }
+ )
+ assert isinstance(llm, NovitaChat)
+ assert llm.model == "zai-org/glm-5"
+
+ @patch("openai.OpenAI")
+ def test_get_llm_novita_custom_base_url(self, mock_openai_cls):
+ from leann.chat import NovitaChat, get_llm
+
+ llm = get_llm(
+ {
+ "type": "novita",
+ "api_key": "test-key",
+ "base_url": "https://custom.novita.url/v1",
+ }
+ )
+ assert isinstance(llm, NovitaChat)
+ assert llm.base_url == "https://custom.novita.url/v1"
+
+
+ not os.getenv("NOVITA_API_KEY"),
+ reason="NOVITA_API_KEY not set; skipping live API test",
+)
+class TestNovitaLiveAPI:
+ """Live API tests for Novita provider (requires NOVITA_API_KEY)."""
+
+ def test_novita_kimi_k25_live(self):
+ from leann.chat import NovitaChat
+
+ chat = NovitaChat(model="moonshotai/kimi-k2.5")
+ response = chat.ask("Say hello in one word.", max_tokens=10)
+ assert isinstance(response, str)
+ assert len(response) > 0
+
+ def test_novita_via_get_llm_live(self):
+ from leann.chat import get_llm
+
+ llm = get_llm({"type": "novita"})
+ response = llm.ask("What is 1+1? Reply with just the number.", max_tokens=10)
+ assert isinstance(response, str)
+ assert len(response) > 0