Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
LiteLLM Rerank API Client.
"""
# For logging, use Python's built-in logging
import time
from typing import Any, List, Optional
from openviking.models.rerank.base import RerankBase
from openviking_cli.utils import get_logger
logger = get_logger(__name__)
def _result_field(item: Any, name: str, default: Any = None) -> Any:
"""Read a field from a litellm rerank result item.
LiteLLM returns result items as objects for some providers (e.g. Cohere)
and as plain dicts for others (e.g. Voyage), so support both shapes.
"""
if isinstance(item, dict):
return item.get(name, default)
return getattr(item, name, default)
class LiteLLMRerankClient(RerankBase):
"""
LiteLLM rerank API client.
"""
def __init__(self, api_key: Optional[str], api_base: Optional[str], model_name: str):
"""
Initialize LiteLLM rerank client.
Args:
api_key: API key for LiteLLM providers (optional, can come from env)
api_base: API base for LiteLLM providers (optional, can come from env)
model_name: Model name to use for reranking
"""
super().__init__()
self.api_key = api_key
self.api_base = api_base
self.model_name = model_name
self.provider = "litellm"
def rerank_batch(self, query: str, documents: List[str]) -> Optional[List[float]]:
"""
Batch rerank documents against a query.
Args:
query: Query text
documents: List of document texts to rank
Returns:
List of rerank scores for each document (same order as input),
or None when rerank fails and the caller should fall back
"""
if not documents:
return []
try:
import litellm
started = time.monotonic()
response = litellm.rerank(
model=self.model_name,
query=query,
documents=documents,
api_key=self.api_key,
api_base=self.api_base,
)
# Update token usage tracking (estimate from response or input)
response_dict = (
response.model_dump() if hasattr(response, "model_dump") else response.__dict__
)
self._extract_and_update_token_usage(
response_dict,
query,
documents,
duration_seconds=time.monotonic() - started,
)
results = response.results
if not results:
logger.warning(f"[LiteLLMRerankClient] Unexpected response format: {response}")
return None
if len(results) != len(documents):
logger.warning(
"[LiteLLMRerankClient] Unexpected rerank result length: expected=%s actual=%s",
len(documents),
len(results),
)
return None
for item in results:
idx = _result_field(item, "index")
if idx is None or not (0 <= idx < len(documents)):
logger.warning(
"[LiteLLMRerankClient] Out-of-bounds or missing index in result: %s", item
)
return None
# Results may not be in original order — sort by index
sorted_results = sorted(results, key=lambda x: _result_field(x, "index"))
scores = [_result_field(item, "relevance_score", 0.0) for item in sorted_results]
logger.debug(f"[LiteLLMRerankClient] Reranked {len(documents)} documents")
return scores
except Exception as e:
logger.error(f"[LiteLLMRerankClient] Rerank failed: {e}")
return None
@classmethod
def from_config(cls, config) -> Optional["LiteLLMRerankClient"]:
"""
Create LiteLLMRerankClient from RerankConfig.
Args:
config: RerankConfig instance with provider='litellm'
Returns:
LiteLLMRerankClient instance or None if config is not available
"""
if not config or not config.is_available():
return None
return cls(
api_key=config.api_key,
api_base=config.api_base,
model_name=config.model,
)