Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(retrieve): bound reranker input size (#3289)
huangruiteng19ca274a24f9
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 971b4b4d3..3b4906ca7 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -850,6 +850,7 @@ Reranking model for search result refinement. Supports VikingDB (Volcengine), Co"api_base": "https://dashscope.aliyuncs.com/compatible-api/v1/reranks","model": "qwen3-vl-rerank","timeout": 120,+ "max_input_tokens": 2048,"threshold": 0.1}}@@ -867,6 +868,7 @@ Reranking model for search result refinement. Supports VikingDB (Volcengine), Co| `api_base` | str | Endpoint URL (for `openai` provider) || `model` | str | Model name (for `openai` providers) || `timeout` | float | HTTP request timeout in seconds for OpenAI-compatible providers. Increase for slow or cold-starting local rerank servers. Default: `30.0` |+| `max_input_tokens` | int | Maximum estimated raw-text tokens in each query-document pair sent to the reranker. Oversized inputs retain their beginning and end. `0` disables truncation. Default: `0` || `threshold` | float | Score threshold between `0.0` and `1.0`; results below this are filtered out. Default: `0.1` || `extra_headers` | object | Custom HTTP headers (for OpenAI-compatible providers, optional) |@@ -1640,6 +1642,7 @@ For detailed encryption explanations, see [Data Encryption](../concepts/10-encry"api_key": "string","model": "string","api_base": "string",+ "max_input_tokens": 0,"threshold": 0.1,"extra_headers": {}},diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex 77a7c4523..6da6ac952 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -819,6 +819,7 @@ AST 提取支持:Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。"api_base": "https://dashscope.aliyuncs.com/compatible-api/v1/reranks","model": "qwen3-vl-rerank","timeout": 120,+ "max_input_tokens": 2048,"threshold": 0.1}}@@ -836,6 +837,7 @@ AST 提取支持:Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。| `api_base` | str | 接口地址(用于 `openai` 提供方) || `model` | str | 模型名称(用于 `openai` 提供方) || `timeout` | float | OpenAI 兼容 provider 的 HTTP 请求超时时间,单位为秒。对于较慢或冷启动的本地 rerank 服务可适当增大。默认:`30.0` |+| `max_input_tokens` | int | 每个 query-document 对发送给 reranker 的最大估算原始文本 token 数;超长输入会保留开头和结尾。`0` 表示不截断。默认:`0` || `threshold` | float | 分数阈值,范围为 `0.0` 到 `1.0`。低于此值的结果会被过滤。默认:`0.1` || `extra_headers` | object | 自定义 HTTP 请求头(OpenAI 兼容 provider 可用,可选) |@@ -1607,6 +1609,7 @@ Task 记录文件位于所属账号的系统目录:"api_key": "string","model": "string","api_base": "string",+ "max_input_tokens": 0,"threshold": 0.1,"extra_headers": {}},diff --git a/openviking/models/rerank/base.py b/openviking/models/rerank/base.pyindex 89b4e7b60..014aee6f1 100644--- a/openviking/models/rerank/base.py+++ b/openviking/models/rerank/base.py@@ -9,6 +9,7 @@ Provides common token usage tracking functionality.import loggingfrom typing import Any, Dict+from openviking.utils.token_estimation import estimate_text_tokensfrom openviking_cli.utils import get_logger_token_tracker_instance = None@@ -48,8 +49,8 @@ class RerankBase:self._token_tracker = _get_token_tracker()def _estimate_tokens(self, text: str) -> int:- """Estimate token count for text (rough approximation: 1 token ≈ 4 characters)."""- return max(1, len(text) // 4)+ """Estimate token count with the shared mixed-language fallback."""+ return max(1, estimate_text_tokens(text))def update_token_usage(self,diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.pyindex 011d51efa..2c8c96a5c 100644--- a/openviking/retrieve/hierarchical_retriever.py+++ b/openviking/retrieve/hierarchical_retriever.py@@ -25,6 +25,10 @@ from openviking.storage import VikingDBManager, VikingDBManagerProxyfrom openviking.storage.expr import FilterExprfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.time_utils import parse_iso_datetime+from openviking.utils.token_estimation import (+ estimate_text_tokens,+ truncate_text_to_token_budget,+)from openviking_cli.exceptions import InvalidArgumentErrorfrom openviking_cli.retrieve.types import (ContextType,@@ -71,6 +75,7 @@ class HierarchicalRetriever:self.vector_store = storageself.embedder = embedderself.rerank_config = rerank_config+ self.rerank_max_input_tokens = rerank_config.max_input_tokens if rerank_config else 0self.retrieval_config = retrieval_config or RetrievalConfig()self.hotness_alpha = self.retrieval_config.hotness_alphaself.score_propagation_alpha = self.retrieval_config.score_propagation_alpha@@ -346,16 +351,27 @@ class HierarchicalRetriever:if not self._rerank_client or not documents:return fallback_scores+ rerank_query = queryrerank_documents = [(index, document) for index, document in enumerate(documents) if document.strip()]if not rerank_documents:return fallback_scores+ if self.rerank_max_input_tokens > 0:+ max_query_tokens = self.rerank_max_input_tokens * 3 // 4+ if estimate_text_tokens(query) > max_query_tokens:+ rerank_query = truncate_text_to_token_budget(query, max_query_tokens)+ document_tokens = self.rerank_max_input_tokens - estimate_text_tokens(rerank_query)+ rerank_documents = [+ (index, truncate_text_to_token_budget(document, document_tokens))+ for index, document in rerank_documents+ ]+try:scores = await asyncio.to_thread(self._rerank_client.rerank_batch,- query,+ rerank_query,[document for _, document in rerank_documents],)except Exception as e:diff --git a/openviking/utils/token_estimation.py b/openviking/utils/token_estimation.pyindex d1d2465eb..16caf2fe2 100644--- a/openviking/utils/token_estimation.py+++ b/openviking/utils/token_estimation.py@@ -47,3 +47,44 @@ def estimate_serialized_tokens(value: Any) -> int:if isinstance(value, str):return estimate_text_tokens(value)return estimate_text_tokens(str(value))+++def truncate_text_to_token_budget(+ text: str,+ max_tokens: int,+ *,+ marker: str = "\n…\n",+ head_ratio: float = 0.75,+) -> str:+ """Fit text to a conservative token budget while retaining its head and tail."""+ if not text or estimate_text_tokens(text) <= max_tokens:+ return text+ if max_tokens <= 0:+ return ""++ marker_tokens = estimate_text_tokens(marker)+ if marker_tokens >= max_tokens:+ marker = ""+ marker_tokens = 0++ content_tokens = max_tokens - marker_tokens+ bounded_head_ratio = min(max(head_ratio, 0.0), 1.0)+ head_tokens = int(content_tokens * bounded_head_ratio)+ tail_tokens = content_tokens - head_tokens++ def _prefix_length(value: str, budget: int) -> int:+ low = 0+ high = len(value)+ while low < high:+ mid = (low + high + 1) // 2+ if estimate_text_tokens(value[:mid]) <= budget:+ low = mid+ else:+ high = mid - 1+ return low++ head_end = _prefix_length(text, head_tokens)+ remaining = text[head_end:]+ tail_length = _prefix_length(remaining[::-1], tail_tokens)+ tail = remaining[len(remaining) - tail_length :] if tail_length else ""+ return text[:head_end].rstrip() + marker + tail.lstrip()diff --git a/openviking_cli/utils/config/rerank_config.py b/openviking_cli/utils/config/rerank_config.pyindex 6cc5803ad..bd26999d3 100644--- a/openviking_cli/utils/config/rerank_config.py+++ b/openviking_cli/utils/config/rerank_config.py@@ -47,6 +47,15 @@ class RerankConfig(BaseModel):default=0.1, description="Relevance threshold (score > threshold is relevant)")+ max_input_tokens: int = Field(+ default=0,+ ge=0,+ description=(+ "Maximum estimated raw-text tokens for each query-document pair sent to "+ "the rerank provider; 0 disables truncation"+ ),+ )+model_config = {"extra": "forbid"}def _effective_provider(self) -> Optional[str]:@@ -63,6 +72,9 @@ class RerankConfig(BaseModel):@model_validator(mode="after")def validate_provider_fields(self) -> "RerankConfig":+ if 0 < self.max_input_tokens < 128:+ raise ValueError("Rerank max_input_tokens must be 0 or at least 128")+provider = self._effective_provider()if provider and provider not in ["vikingdb", "cohere", "openai", "litellm"]:raise ValueError(diff --git a/tests/retrieve/test_hierarchical_retriever_rerank.py b/tests/retrieve/test_hierarchical_retriever_rerank.pyindex 49c52356d..1ed0a0b34 100644--- a/tests/retrieve/test_hierarchical_retriever_rerank.py+++ b/tests/retrieve/test_hierarchical_retriever_rerank.py@@ -11,6 +11,7 @@ import pytestfrom openviking.retrieve.hierarchical_retriever import HierarchicalRetriever, RetrieverModefrom openviking.server.identity import RequestContext, Role+from openviking.utils.token_estimation import estimate_text_tokensfrom openviking_cli.retrieve.types import ContextType, TypedQueryfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils.config import RerankConfig, RetrievalConfig@@ -238,6 +239,12 @@ def test_retriever_initializes_rerank_client(monkeypatch):assert retriever._rerank_client is fake_client+def test_rerank_max_input_tokens_accepts_zero_or_at_least_128():+ assert RerankConfig(max_input_tokens=0).max_input_tokens == 0+ with pytest.raises(ValueError, match="max_input_tokens"):+ RerankConfig(max_input_tokens=127)++@pytest.mark.asyncioasync def test_retrieve_uses_rerank_scores_in_thinking_mode(monkeypatch):fake_client = FakeRerankClient([0.95, 0.05, 0.11, 0.95])@@ -288,6 +295,61 @@ async def test_rerank_scores_preserves_fallbacks_for_empty_documents(monkeypatchassert fake_client.calls == [("hello", ["root A", "root D"])]+async def test_rerank_scores_does_not_truncate_by_default(monkeypatch):+ oversized_document = "summary-start " + ("填充内容" * 600) + " relevant-tail"+ fake_client = FakeRerankClient([0.95])+ monkeypatch.setattr(+ "openviking.retrieve.hierarchical_retriever.RerankClient.from_config",+ lambda config: fake_client,+ )++ retriever = HierarchicalRetriever(+ storage=DummyStorage(),+ embedder=DummyEmbedder(),+ rerank_config=RerankConfig(ak="ak", sk="sk"),+ )++ await retriever._rerank_scores("query", [oversized_document], [0.2])++ assert retriever.rerank_max_input_tokens == 0+ assert fake_client.calls == [("query", [oversized_document])]+++ "oversized_document",+ [+ "summary-start " + ("filler " * 600) + " relevant-tail",+ "摘要开头" + ("填充内容" * 600) + "相关结论",+ ],+)+async def test_rerank_scores_bounds_oversized_documents_and_preserves_tail(+ monkeypatch, oversized_document+):+ fake_client = FakeRerankClient([0.95])+ monkeypatch.setattr(+ "openviking.retrieve.hierarchical_retriever.RerankClient.from_config",+ lambda config: fake_client,+ )++ retriever = HierarchicalRetriever(+ storage=DummyStorage(),+ embedder=DummyEmbedder(),+ rerank_config=RerankConfig(ak="ak", sk="sk", max_input_tokens=128),+ )++ scores = await retriever._rerank_scores("query", [oversized_document], [0.2])++ assert scores == [0.95]+ rerank_query, rerank_documents = fake_client.calls[0]+ bounded_document = rerank_documents[0]+ assert estimate_text_tokens(rerank_query) + estimate_text_tokens(bounded_document) <= 128+ assert "summary-start" in bounded_document or "摘要开头" in bounded_document+ assert "relevant-tail" in bounded_document or "相关结论" in bounded_document+ assert bounded_document != oversized_document++@pytest.mark.asyncioasync def test_retrieve_falls_back_to_vector_scores_when_rerank_returns_none(monkeypatch):class NoneRerankClient(FakeRerankClient):diff --git a/tests/test_token_estimation.py b/tests/test_token_estimation.pyindex 82fba2e67..97a4f1464 100644--- a/tests/test_token_estimation.py+++ b/tests/test_token_estimation.py@@ -5,6 +5,10 @@from openviking.message import Message, TextPartfrom openviking.session import Session+from openviking.utils.token_estimation import (+ estimate_text_tokens,+ truncate_text_to_token_budget,+)def test_message_estimated_tokens_is_cjk_aware():@@ -14,6 +18,16 @@ def test_message_estimated_tokens_is_cjk_aware():assert msg.estimated_tokens == 6+def test_truncate_text_to_token_budget_preserves_head_and_tail():+ text = "summary-start " + ("填充内容" * 100) + " relevant-tail"++ truncated = truncate_text_to_token_budget(text, 32)++ assert estimate_text_tokens(truncated) <= 32+ assert truncated.startswith("summary-start")+ assert truncated.endswith("relevant-tail")++async def test_archive_overview_tokens_do_not_trust_stale_low_metadata():class FakeFS:async def read_file(self, uri, ctx=None):