Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(retrieve): bound reranker input size (#3289)

huangruiteng19ca274a24f9
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 971b4b4d3..3b4906ca7 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -850,6 +850,7 @@ Reranking model for search result refinement. Supports VikingDB (Volcengine), Co
"api_base": "https://dashscope.aliyuncs.com/compatible-api/v1/reranks",
"model": "qwen3-vl-rerank",
"timeout": 120,
+ "max_input_tokens": 2048,
"threshold": 0.1
}
}
@@ -867,6 +868,7 @@ Reranking model for search result refinement. Supports VikingDB (Volcengine), Co
| `api_base` | str | Endpoint URL (for `openai` provider) |
| `model` | str | Model name (for `openai` providers) |
| `timeout` | float | HTTP request timeout in seconds for OpenAI-compatible providers. Increase for slow or cold-starting local rerank servers. Default: `30.0` |
+| `max_input_tokens` | int | Maximum estimated raw-text tokens in each query-document pair sent to the reranker. Oversized inputs retain their beginning and end. `0` disables truncation. Default: `0` |
| `threshold` | float | Score threshold between `0.0` and `1.0`; results below this are filtered out. Default: `0.1` |
| `extra_headers` | object | Custom HTTP headers (for OpenAI-compatible providers, optional) |
@@ -1640,6 +1642,7 @@ For detailed encryption explanations, see [Data Encryption](../concepts/10-encry
"api_key": "string",
"model": "string",
"api_base": "string",
+ "max_input_tokens": 0,
"threshold": 0.1,
"extra_headers": {}
},
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index 77a7c4523..6da6ac952 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -819,6 +819,7 @@ AST 提取支持:Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。
"api_base": "https://dashscope.aliyuncs.com/compatible-api/v1/reranks",
"model": "qwen3-vl-rerank",
"timeout": 120,
+ "max_input_tokens": 2048,
"threshold": 0.1
}
}
@@ -836,6 +837,7 @@ AST 提取支持:Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。
| `api_base` | str | 接口地址(用于 `openai` 提供方) |
| `model` | str | 模型名称(用于 `openai` 提供方) |
| `timeout` | float | OpenAI 兼容 provider 的 HTTP 请求超时时间,单位为秒。对于较慢或冷启动的本地 rerank 服务可适当增大。默认:`30.0` |
+| `max_input_tokens` | int | 每个 query-document 对发送给 reranker 的最大估算原始文本 token 数;超长输入会保留开头和结尾。`0` 表示不截断。默认:`0` |
| `threshold` | float | 分数阈值,范围为 `0.0` 到 `1.0`。低于此值的结果会被过滤。默认:`0.1` |
| `extra_headers` | object | 自定义 HTTP 请求头(OpenAI 兼容 provider 可用,可选) |
@@ -1607,6 +1609,7 @@ Task 记录文件位于所属账号的系统目录:
"api_key": "string",
"model": "string",
"api_base": "string",
+ "max_input_tokens": 0,
"threshold": 0.1,
"extra_headers": {}
},
diff --git a/openviking/models/rerank/base.py b/openviking/models/rerank/base.py
index 89b4e7b60..014aee6f1 100644
--- a/openviking/models/rerank/base.py
+++ b/openviking/models/rerank/base.py
@@ -9,6 +9,7 @@ Provides common token usage tracking functionality.
import logging
from typing import Any, Dict
+from openviking.utils.token_estimation import estimate_text_tokens
from openviking_cli.utils import get_logger
_token_tracker_instance = None
@@ -48,8 +49,8 @@ class RerankBase:
self._token_tracker = _get_token_tracker()
def _estimate_tokens(self, text: str) -> int:
- """Estimate token count for text (rough approximation: 1 token ≈ 4 characters)."""
- return max(1, len(text) // 4)
+ """Estimate token count with the shared mixed-language fallback."""
+ return max(1, estimate_text_tokens(text))
def update_token_usage(
self,
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py
index 011d51efa..2c8c96a5c 100644
--- a/openviking/retrieve/hierarchical_retriever.py
+++ b/openviking/retrieve/hierarchical_retriever.py
@@ -25,6 +25,10 @@ from openviking.storage import VikingDBManager, VikingDBManagerProxy
from openviking.storage.expr import FilterExpr
from openviking.telemetry import get_current_telemetry
from openviking.utils.time_utils import parse_iso_datetime
+from openviking.utils.token_estimation import (
+ estimate_text_tokens,
+ truncate_text_to_token_budget,
+)
from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.retrieve.types import (
ContextType,
@@ -71,6 +75,7 @@ class HierarchicalRetriever:
self.vector_store = storage
self.embedder = embedder
self.rerank_config = rerank_config
+ self.rerank_max_input_tokens = rerank_config.max_input_tokens if rerank_config else 0
self.retrieval_config = retrieval_config or RetrievalConfig()
self.hotness_alpha = self.retrieval_config.hotness_alpha
self.score_propagation_alpha = self.retrieval_config.score_propagation_alpha
@@ -346,16 +351,27 @@ class HierarchicalRetriever:
if not self._rerank_client or not documents:
return fallback_scores
+ rerank_query = query
rerank_documents = [
(index, document) for index, document in enumerate(documents) if document.strip()
]
if not rerank_documents:
return fallback_scores
+ if self.rerank_max_input_tokens > 0:
+ max_query_tokens = self.rerank_max_input_tokens * 3 // 4
+ if estimate_text_tokens(query) > max_query_tokens:
+ rerank_query = truncate_text_to_token_budget(query, max_query_tokens)
+ document_tokens = self.rerank_max_input_tokens - estimate_text_tokens(rerank_query)
+ rerank_documents = [
+ (index, truncate_text_to_token_budget(document, document_tokens))
+ for index, document in rerank_documents
+ ]
+
try:
scores = await asyncio.to_thread(
self._rerank_client.rerank_batch,
- query,
+ rerank_query,
[document for _, document in rerank_documents],
)
except Exception as e:
diff --git a/openviking/utils/token_estimation.py b/openviking/utils/token_estimation.py
index d1d2465eb..16caf2fe2 100644
--- a/openviking/utils/token_estimation.py
+++ b/openviking/utils/token_estimation.py
@@ -47,3 +47,44 @@ def estimate_serialized_tokens(value: Any) -> int:
if isinstance(value, str):
return estimate_text_tokens(value)
return estimate_text_tokens(str(value))
+
+
+def truncate_text_to_token_budget(
+ text: str,
+ max_tokens: int,
+ *,
+ marker: str = "\n…\n",
+ head_ratio: float = 0.75,
+) -> str:
+ """Fit text to a conservative token budget while retaining its head and tail."""
+ if not text or estimate_text_tokens(text) <= max_tokens:
+ return text
+ if max_tokens <= 0:
+ return ""
+
+ marker_tokens = estimate_text_tokens(marker)
+ if marker_tokens >= max_tokens:
+ marker = ""
+ marker_tokens = 0
+
+ content_tokens = max_tokens - marker_tokens
+ bounded_head_ratio = min(max(head_ratio, 0.0), 1.0)
+ head_tokens = int(content_tokens * bounded_head_ratio)
+ tail_tokens = content_tokens - head_tokens
+
+ def _prefix_length(value: str, budget: int) -> int:
+ low = 0
+ high = len(value)
+ while low < high:
+ mid = (low + high + 1) // 2
+ if estimate_text_tokens(value[:mid]) <= budget:
+ low = mid
+ else:
+ high = mid - 1
+ return low
+
+ head_end = _prefix_length(text, head_tokens)
+ remaining = text[head_end:]
+ tail_length = _prefix_length(remaining[::-1], tail_tokens)
+ tail = remaining[len(remaining) - tail_length :] if tail_length else ""
+ return text[:head_end].rstrip() + marker + tail.lstrip()
diff --git a/openviking_cli/utils/config/rerank_config.py b/openviking_cli/utils/config/rerank_config.py
index 6cc5803ad..bd26999d3 100644
--- a/openviking_cli/utils/config/rerank_config.py
+++ b/openviking_cli/utils/config/rerank_config.py
@@ -47,6 +47,15 @@ class RerankConfig(BaseModel):
default=0.1, description="Relevance threshold (score > threshold is relevant)"
)
+ max_input_tokens: int = Field(
+ default=0,
+ ge=0,
+ description=(
+ "Maximum estimated raw-text tokens for each query-document pair sent to "
+ "the rerank provider; 0 disables truncation"
+ ),
+ )
+
model_config = {"extra": "forbid"}
def _effective_provider(self) -> Optional[str]:
@@ -63,6 +72,9 @@ class RerankConfig(BaseModel):
@model_validator(mode="after")
def validate_provider_fields(self) -> "RerankConfig":
+ if 0 < self.max_input_tokens < 128:
+ raise ValueError("Rerank max_input_tokens must be 0 or at least 128")
+
provider = self._effective_provider()
if provider and provider not in ["vikingdb", "cohere", "openai", "litellm"]:
raise ValueError(
diff --git a/tests/retrieve/test_hierarchical_retriever_rerank.py b/tests/retrieve/test_hierarchical_retriever_rerank.py
index 49c52356d..1ed0a0b34 100644
--- a/tests/retrieve/test_hierarchical_retriever_rerank.py
+++ b/tests/retrieve/test_hierarchical_retriever_rerank.py
@@ -11,6 +11,7 @@ import pytest
from openviking.retrieve.hierarchical_retriever import HierarchicalRetriever, RetrieverMode
from openviking.server.identity import RequestContext, Role
+from openviking.utils.token_estimation import estimate_text_tokens
from openviking_cli.retrieve.types import ContextType, TypedQuery
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.config import RerankConfig, RetrievalConfig
@@ -238,6 +239,12 @@ def test_retriever_initializes_rerank_client(monkeypatch):
assert retriever._rerank_client is fake_client
+def test_rerank_max_input_tokens_accepts_zero_or_at_least_128():
+ assert RerankConfig(max_input_tokens=0).max_input_tokens == 0
+ with pytest.raises(ValueError, match="max_input_tokens"):
+ RerankConfig(max_input_tokens=127)
+
+
@pytest.mark.asyncio
async def test_retrieve_uses_rerank_scores_in_thinking_mode(monkeypatch):
fake_client = FakeRerankClient([0.95, 0.05, 0.11, 0.95])
@@ -288,6 +295,61 @@ async def test_rerank_scores_preserves_fallbacks_for_empty_documents(monkeypatch
assert fake_client.calls == [("hello", ["root A", "root D"])]
+async def test_rerank_scores_does_not_truncate_by_default(monkeypatch):
+ oversized_document = "summary-start " + ("填充内容" * 600) + " relevant-tail"
+ fake_client = FakeRerankClient([0.95])
+ monkeypatch.setattr(
+ "openviking.retrieve.hierarchical_retriever.RerankClient.from_config",
+ lambda config: fake_client,
+ )
+
+ retriever = HierarchicalRetriever(
+ storage=DummyStorage(),
+ embedder=DummyEmbedder(),
+ rerank_config=RerankConfig(ak="ak", sk="sk"),
+ )
+
+ await retriever._rerank_scores("query", [oversized_document], [0.2])
+
+ assert retriever.rerank_max_input_tokens == 0
+ assert fake_client.calls == [("query", [oversized_document])]
+
+
+ "oversized_document",
+ [
+ "summary-start " + ("filler " * 600) + " relevant-tail",
+ "摘要开头" + ("填充内容" * 600) + "相关结论",
+ ],
+)
+async def test_rerank_scores_bounds_oversized_documents_and_preserves_tail(
+ monkeypatch, oversized_document
+):
+ fake_client = FakeRerankClient([0.95])
+ monkeypatch.setattr(
+ "openviking.retrieve.hierarchical_retriever.RerankClient.from_config",
+ lambda config: fake_client,
+ )
+
+ retriever = HierarchicalRetriever(
+ storage=DummyStorage(),
+ embedder=DummyEmbedder(),
+ rerank_config=RerankConfig(ak="ak", sk="sk", max_input_tokens=128),
+ )
+
+ scores = await retriever._rerank_scores("query", [oversized_document], [0.2])
+
+ assert scores == [0.95]
+ rerank_query, rerank_documents = fake_client.calls[0]
+ bounded_document = rerank_documents[0]
+ assert estimate_text_tokens(rerank_query) + estimate_text_tokens(bounded_document) <= 128
+ assert "summary-start" in bounded_document or "摘要开头" in bounded_document
+ assert "relevant-tail" in bounded_document or "相关结论" in bounded_document
+ assert bounded_document != oversized_document
+
+
@pytest.mark.asyncio
async def test_retrieve_falls_back_to_vector_scores_when_rerank_returns_none(monkeypatch):
class NoneRerankClient(FakeRerankClient):
diff --git a/tests/test_token_estimation.py b/tests/test_token_estimation.py
index 82fba2e67..97a4f1464 100644
--- a/tests/test_token_estimation.py
+++ b/tests/test_token_estimation.py
@@ -5,6 +5,10 @@
from openviking.message import Message, TextPart
from openviking.session import Session
+from openviking.utils.token_estimation import (
+ estimate_text_tokens,
+ truncate_text_to_token_budget,
+)
def test_message_estimated_tokens_is_cjk_aware():
@@ -14,6 +18,16 @@ def test_message_estimated_tokens_is_cjk_aware():
assert msg.estimated_tokens == 6
+def test_truncate_text_to_token_budget_preserves_head_and_tail():
+ text = "summary-start " + ("填充内容" * 100) + " relevant-tail"
+
+ truncated = truncate_text_to_token_budget(text, 32)
+
+ assert estimate_text_tokens(truncated) <= 32
+ assert truncated.startswith("summary-start")
+ assert truncated.endswith("relevant-tail")
+
+
async def test_archive_overview_tokens_do_not_trust_stale_low_metadata():
class FakeFS:
async def read_file(self, uri, ctx=None):