Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(rerank): add litellm as rerank provider (#888)

Matt Van Hornc10f6d9df09f
diff --git a/openviking_cli/utils/config/rerank_config.py b/openviking_cli/utils/config/rerank_config.py
index 91c0b0236..0a4f3928f 100644
--- a/openviking_cli/utils/config/rerank_config.py
+++ b/openviking_cli/utils/config/rerank_config.py
@@ -8,7 +8,9 @@ from pydantic import BaseModel, Field, model_validator
class RerankConfig(BaseModel):
"""Configuration for rerank API (VikingDB or OpenAI-compatible providers)."""
- provider: str = Field(default="vikingdb", description="Rerank provider: 'vikingdb' or 'openai'")
+ provider: str = Field(
+ default="vikingdb", description="Rerank provider: 'vikingdb', 'openai', or 'litellm'"
+ )
# VikingDB fields
ak: Optional[str] = Field(default=None, description="VikingDB Access Key")
@@ -36,7 +38,7 @@ class RerankConfig(BaseModel):
@model_validator(mode="after")
def validate_provider_fields(self) -> "RerankConfig":
- allowed = ["vikingdb", "openai"]
+ allowed = ["vikingdb", "openai", "litellm"]
if self.provider not in allowed:
raise ValueError(f"Rerank provider must be one of {allowed}, got '{self.provider}'")
if self.provider == "openai":
@@ -44,10 +46,15 @@ class RerankConfig(BaseModel):
raise ValueError(
"OpenAI-compatible rerank provider requires 'api_key' and 'api_base'"
)
+ if self.provider == "litellm":
+ if not self.model:
+ raise ValueError("LiteLLM rerank provider requires 'model'")
return self
def is_available(self) -> bool:
"""Check if rerank is configured."""
if self.provider == "openai":
return self.api_key is not None and self.api_base is not None
+ if self.provider == "litellm":
+ return self.model is not None
return self.ak is not None and self.sk is not None
diff --git a/openviking_cli/utils/rerank.py b/openviking_cli/utils/rerank.py
index 8ca906023..6566fc985 100644
--- a/openviking_cli/utils/rerank.py
+++ b/openviking_cli/utils/rerank.py
@@ -156,6 +156,11 @@ class RerankClient:
if not config or not config.is_available():
return None
+ if config.provider == "litellm":
+ from openviking_cli.utils.rerank_litellm import LiteLLMRerankClient
+
+ return LiteLLMRerankClient.from_config(config)
+
if config.provider == "openai":
from openviking_cli.utils.rerank_openai import OpenAIRerankClient
diff --git a/openviking_cli/utils/rerank_litellm.py b/openviking_cli/utils/rerank_litellm.py
new file mode 100644
index 000000000..9ef6efa50
--- /dev/null
+++ b/openviking_cli/utils/rerank_litellm.py
@@ -0,0 +1,107 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""
+LiteLLM Rerank API Client.
+"""
+
+from typing import List, Optional
+
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class LiteLLMRerankClient:
+ """
+ LiteLLM rerank API client.
+ """
+
+ def __init__(self, api_key: Optional[str], api_base: Optional[str], model_name: str):
+ """
+ Initialize LiteLLM rerank client.
+
+ Args:
+ api_key: API key for LiteLLM providers (optional, can come from env)
+ api_base: API base for LiteLLM providers (optional, can come from env)
+ model_name: Model name to use for reranking
+ """
+ self.api_key = api_key
+ self.api_base = api_base
+ self.model_name = model_name
+
+ def rerank_batch(self, query: str, documents: List[str]) -> Optional[List[float]]:
+ """
+ Batch rerank documents against a query.
+
+ Args:
+ query: Query text
+ documents: List of document texts to rank
+
+ Returns:
+ List of rerank scores for each document (same order as input),
+ or None when rerank fails and the caller should fall back
+ """
+ if not documents:
+ return []
+
+ try:
+ import litellm
+
+ response = litellm.rerank(
+ model=self.model_name,
+ query=query,
+ documents=[{"text": d} for d in documents],
+ api_key=self.api_key,
+ api_base=self.api_base,
+ )
+
+ results = response.results
+ if not results:
+ logger.warning(f"[LiteLLMRerankClient] Unexpected response format: {response}")
+ return None
+
+ if len(results) != len(documents):
+ logger.warning(
+ "[LiteLLMRerankClient] Unexpected rerank result length: expected=%s actual=%s",
+ len(documents),
+ len(results),
+ )
+ return None
+
+ for item in results:
+ idx = getattr(item, "index", None)
+ if idx is None or not (0 <= idx < len(documents)):
+ logger.warning(
+ "[LiteLLMRerankClient] Out-of-bounds or missing index in result: %s", item
+ )
+ return None
+
+ # Results may not be in original order — sort by index
+ sorted_results = sorted(results, key=lambda x: x.index)
+ scores = [getattr(item, "relevance_score", 0.0) for item in sorted_results]
+
+ logger.debug(f"[LiteLLMRerankClient] Reranked {len(documents)} documents")
+ return scores
+
+ except Exception as e:
+ logger.error(f"[LiteLLMRerankClient] Rerank failed: {e}")
+ return None
+
+ @classmethod
+ def from_config(cls, config) -> Optional["LiteLLMRerankClient"]:
+ """
+ Create LiteLLMRerankClient from RerankConfig.
+
+ Args:
+ config: RerankConfig instance with provider='litellm'
+
+ Returns:
+ LiteLLMRerankClient instance or None if config is not available
+ """
+ if not config or not config.is_available():
+ return None
+ return cls(
+ api_key=config.api_key,
+ api_base=config.api_base,
+ model_name=config.model,
+ )