Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor: Add RERANK_TOKEN_LIMIT_MODE = truncate (default) / passthrough / raise_error (#19404)

Wang Qia13593097bd5
diff --git a/api/db/services/tenant_llm_service.py b/api/db/services/tenant_llm_service.py
index f5153a5a10..42eafcdd06 100644
--- a/api/db/services/tenant_llm_service.py
+++ b/api/db/services/tenant_llm_service.py
@@ -196,7 +196,7 @@ class TenantLLMService(CommonService):
if model_config["llm_factory"] not in RerankModel:
logging.error("Factory not in rerank model. Supported factories: %s", list(RerankModel.keys()))
return None
- return RerankModel[model_config["llm_factory"]](api_key, model_config["llm_name"], base_url=model_config["api_base"])
+ return RerankModel[model_config["llm_factory"]](api_key, model_config["llm_name"], base_url=model_config["api_base"], max_token=model_config.get("max_tokens"))
elif model_config["model_type"] == LLMType.VISION.value:
if model_config["llm_factory"] not in CvModel:
diff --git a/docker/.env b/docker/.env
index 120f6b6150..67ecec3965 100644
--- a/docker/.env
+++ b/docker/.env
@@ -312,6 +312,13 @@ DOC_BULK_SIZE=${DOC_BULK_SIZE:-4}
# Defaults to 16 if EMBEDDING_BATCH_SIZE is not set in the environment.
EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-16}
+# Controls how rerank inputs that exceed the configured model token limit are handled.
+# Available options:
+# - `truncate` (default): truncate each document so its tokens plus the query tokens fit the limit.
+# - `passthrough`: send the query and documents unchanged and let the rerank provider handle oversized inputs.
+# - `raise_error`: reject the request before calling the provider if a query document pair exceeds the limit.
+RERANK_TOKEN_LIMIT_MODE=${RERANK_TOKEN_LIMIT_MODE:-truncate}
+
# Log level for the RAGFlow's own and imported packages.
# Available levels:
# - `DEBUG`
diff --git a/rag/llm/rerank_model.py b/rag/llm/rerank_model.py
index b026980101..f474182005 100644
--- a/rag/llm/rerank_model.py
+++ b/rag/llm/rerank_model.py
@@ -16,6 +16,7 @@
import json
import logging
import math
+import os
import time
from abc import ABC
from urllib.parse import urljoin
@@ -30,18 +31,24 @@ from common.token_utils import num_tokens_from_string, truncate, total_token_cou
from rag.llm.mws_utils import mws_api_url, require_mws_token
from rag.utils.url_utils import append_api_path, ensure_v1
+MAX_RERANK_TOKEN = 8196
+RERANK_TOKEN_LIMIT_MODE = os.getenv("RERANK_TOKEN_LIMIT_MODE", "truncate").strip().lower()
+if RERANK_TOKEN_LIMIT_MODE not in {"truncate", "passthrough", "raise_error"}:
+ raise ValueError(f"Invalid RERANK_TOKEN_LIMIT_MODE {RERANK_TOKEN_LIMIT_MODE!r}; expected 'truncate', 'passthrough', or 'raise_error'")
+
class Base(ABC):
- def __init__(self, key, model_name, **kwargs):
- pass
+ def __init__(self, key, model_name, max_token=MAX_RERANK_TOKEN, **kwargs):
+ self.max_token = max_token or MAX_RERANK_TOKEN
def similarity(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
"""Score ``texts`` against ``query`` and return ``(rank, token_count)``.
This is the single public entry point shared by every reranker. It
- short-circuits empty input and guarantees the returned scores are
- min-max normalized to ``[0, 1]`` regardless of what the backend emits
- (relevance scores, cosine similarities or raw logits). Downstream
+ short-circuits empty input, applies the configured token-limit policy,
+ and guarantees the returned scores are min-max normalized to ``[0, 1]``
+ regardless of what the backend emits (relevance scores, cosine
+ similarities or raw logits). Downstream
hybrid scoring blends the reranker output with token similarity on a
fixed ``[0, 1]`` scale, so an un-normalized provider (e.g. NVIDIA's
unbounded logits) would otherwise corrupt the final ordering.
@@ -51,6 +58,15 @@ class Base(ABC):
"""
if not query or not texts:
return np.zeros(len(texts) if texts else 0, dtype=float), 0
+ if RERANK_TOKEN_LIMIT_MODE != "passthrough":
+ query_tokens = num_tokens_from_string(query)
+ if RERANK_TOKEN_LIMIT_MODE == "truncate":
+ texts = [truncate(text, max(self.max_token - query_tokens, 0)) for text in texts]
+ else:
+ for index, text in enumerate(texts):
+ input_tokens = query_tokens + num_tokens_from_string(text)
+ if input_tokens > self.max_token:
+ raise ValueError(f"Rerank input at document index {index} has {input_tokens} tokens, exceeding the configured maximum of {self.max_token}")
rank, token_count = self._compute_rank(query, texts)
rank = np.asarray(rank, dtype=float)
if rank.size:
@@ -98,13 +114,13 @@ class Base(ABC):
class JinaRerank(Base):
_FACTORY_NAME = "Jina"
- def __init__(self, key, model_name="jina-reranker-v2-base-multilingual", base_url="https://api.jina.ai/v1/rerank"):
+ def __init__(self, key, model_name="jina-reranker-v2-base-multilingual", base_url="https://api.jina.ai/v1/rerank", max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
self.base_url = base_url or "https://api.jina.ai/v1/rerank"
self.headers = {"Content-Type": "application/json", "Authorization": f"Bearer {key}"}
self.model_name = model_name
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
- texts = [truncate(t, 8196) for t in texts]
data = {"model": self.model_name, "query": query, "documents": texts, "top_n": len(texts)}
response = requests.post(self.base_url, headers=self.headers, json=data, timeout=30)
response.raise_for_status()
@@ -123,17 +139,18 @@ class GreenPTRerank(JinaRerank):
_FACTORY_NAME = "GreenPT"
- def __init__(self, key, model_name="green-rerank", base_url="https://api.greenpt.ai/v1/rerank"):
+ def __init__(self, key, model_name="green-rerank", base_url="https://api.greenpt.ai/v1/rerank", max_token=MAX_RERANK_TOKEN):
endpoint = (base_url or "https://api.greenpt.ai/v1/rerank").rstrip("/")
if not endpoint.endswith("/rerank"):
endpoint += "/rerank"
- super().__init__(key, model_name=model_name, base_url=endpoint)
+ super().__init__(key, model_name=model_name, base_url=endpoint, max_token=max_token)
class XInferenceRerank(Base):
_FACTORY_NAME = "Xinference"
- def __init__(self, key="x", model_name="", base_url=""):
+ def __init__(self, key="x", model_name="", base_url="", max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
if base_url.find("/v1") == -1:
base_url = urljoin(base_url, "/v1/rerank")
if base_url.find("/rerank") == -1:
@@ -145,10 +162,7 @@ class XInferenceRerank(Base):
self.headers["Authorization"] = f"Bearer {key}"
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
- pairs = [(query, truncate(t, 4096)) for t in texts]
- token_count = 0
- for _, t in pairs:
- token_count += num_tokens_from_string(t)
+ token_count = sum(num_tokens_from_string(t) for t in texts)
data = {"model": self.model_name, "query": query, "return_documents": "true", "return_len": "true", "documents": texts}
response = requests.post(self.base_url, headers=self.headers, json=data, timeout=30)
response.raise_for_status()
@@ -165,7 +179,8 @@ class XInferenceRerank(Base):
class LocalAIRerank(Base):
_FACTORY_NAME = "LocalAI"
- def __init__(self, key, model_name, base_url):
+ def __init__(self, key, model_name, base_url, max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
if base_url.find("/rerank") == -1:
self.base_url = urljoin(base_url, "/rerank")
else:
@@ -174,7 +189,6 @@ class LocalAIRerank(Base):
self.model_name = model_name.split("___")[0]
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
- texts = [truncate(t, 500) for t in texts]
data = {
"model": self.model_name,
"query": query,
@@ -199,7 +213,8 @@ class LocalAIRerank(Base):
class NvidiaRerank(Base):
_FACTORY_NAME = "NVIDIA"
- def __init__(self, key, model_name, base_url="https://ai.api.nvidia.com/v1/retrieval/nvidia/"):
+ def __init__(self, key, model_name, base_url="https://ai.api.nvidia.com/v1/retrieval/nvidia/", max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
if not base_url:
base_url = "https://ai.api.nvidia.com/v1/retrieval/nvidia/"
base_url = base_url.rstrip("/") + "/"
@@ -238,7 +253,6 @@ class NvidiaRerank(Base):
"model": self.model_name,
"query": {"text": query},
"passages": [{"text": text} for text in texts],
- "truncate": "END",
"top_n": len(texts),
}
response = requests.post(self.base_url, headers=self.headers, json=data, timeout=30)
@@ -256,8 +270,8 @@ class NvidiaRerank(Base):
class LmStudioRerank(Base):
_FACTORY_NAME = "LM-Studio"
- def __init__(self, key, model_name, base_url, **kwargs):
- pass
+ def __init__(self, key, model_name, base_url, max_token=MAX_RERANK_TOKEN, **kwargs):
+ super().__init__(key, model_name, max_token)
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
raise NotImplementedError("The LmStudioRerank has not been implemented")
@@ -266,7 +280,8 @@ class LmStudioRerank(Base):
class OpenAI_APIRerank(Base):
_FACTORY_NAME = "OpenAI-API-Compatible"
- def __init__(self, key, model_name, base_url):
+ def __init__(self, key, model_name, base_url, max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
normalized_base_url = (base_url or "").strip()
if "/rerank" in normalized_base_url:
self.base_url = normalized_base_url.rstrip("/")
@@ -276,7 +291,6 @@ class OpenAI_APIRerank(Base):
self.model_name = model_name.split("___")[0]
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
- texts = [truncate(t, 500) for t in texts]
data = {
"model": self.model_name,
"query": query,
@@ -303,14 +317,14 @@ class MWSRerank(OpenAI_APIRerank):
_FACTORY_NAME = "MWS"
- def __init__(self, key, model_name, base_url):
+ def __init__(self, key, model_name, base_url, max_token=MAX_RERANK_TOKEN):
"""Initialize reranking access for an MWS project and deployment."""
token = require_mws_token(key)
- super().__init__(token, model_name, mws_api_url(base_url, "cohere/v2/rerank"))
+ super().__init__(token, model_name, mws_api_url(base_url, "cohere/v2/rerank"), max_token)
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
"""Score candidate texts and restore scores to document input order."""
- documents = [truncate(text, 500) for text in texts]
+ documents = texts
log_context = {
"provider": self._FACTORY_NAME,
"operation": "rerank",
@@ -392,7 +406,8 @@ class MWSRerank(OpenAI_APIRerank):
class CoHereRerank(Base):
_FACTORY_NAME = ["Cohere", "VLLM"]
- def __init__(self, key, model_name, base_url=None):
+ def __init__(self, key, model_name, base_url=None, max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
from cohere import Client
client_kwargs = {"api_key": key, "timeout": 30.0}
@@ -426,13 +441,11 @@ class CoHereRerank(Base):
class BedrockRerank(Base):
_FACTORY_NAME = "Bedrock"
- # Hard limits of the bedrock-agent-runtime Rerank API: each document text
- # (RerankTextDocument.text) is capped at 32,000 characters, and a single
- # request accepts at most 1,000 sources / numberOfResults.
- _MAX_DOC_CHARS = 32000
+ # A single request accepts at most 1,000 sources / numberOfResults.
_MAX_SOURCES = 1000
- def __init__(self, key, model_name, **kwargs):
+ def __init__(self, key, model_name, max_token=MAX_RERANK_TOKEN, **kwargs):
+ super().__init__(key, model_name, max_token)
import boto3
from botocore.utils import validate_region_name
@@ -449,12 +462,6 @@ class BedrockRerank(Base):
self.model_name = model_name
# On-demand foundation-model ARN; works for amazon.rerank-v1:0 / cohere.rerank-*.
self.model_arn = f"arn:aws:bedrock:{self.bedrock_region}::foundation-model/{self.model_name}"
- # Per-document truncation guard sized to the model window. Cohere Rerank
- # v3.5 shares a ~4k window between query and document (~2048 for docs);
- # Amazon Rerank v1 handles 32k, but chunks are small so a generous cap
- # just bounds pathological payloads. Bedrock also truncates internally.
- self.doc_max_tokens = 2048 if self.model_name.split(".")[0] == "cohere" else 8192
-
# Rerank lives on the bedrock-agent-runtime service, not bedrock-runtime.
if mode == "access_key_secret":
self.client = boto3.client(
@@ -482,10 +489,6 @@ class BedrockRerank(Base):
raise ValueError(f"Unsupported Bedrock auth_mode: {mode}")
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
- # Truncate to the model token window, then enforce the API's hard 32k-char
- # per-text limit (a longer RerankTextQuery / RerankTextDocument is rejected).
- query = query[: self._MAX_DOC_CHARS]
- texts = [truncate(t, self.doc_max_tokens)[: self._MAX_DOC_CHARS] for t in texts]
# Bedrock does not report token usage; count locally like CoHereRerank.
token_count = num_tokens_from_string(query) + sum(num_tokens_from_string(t) for t in texts)
@@ -536,8 +539,8 @@ class BedrockRerank(Base):
class TogetherAIRerank(Base):
_FACTORY_NAME = "TogetherAI"
- def __init__(self, key, model_name, base_url, **kwargs):
- pass
+ def __init__(self, key, model_name, base_url, max_token=MAX_RERANK_TOKEN, **kwargs):
+ super().__init__(key, model_name, max_token)
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
raise NotImplementedError("The api has not been implemented")
@@ -546,7 +549,8 @@ class TogetherAIRerank(Base):
class SILICONFLOWRerank(Base):
_FACTORY_NAME = "SILICONFLOW"
- def __init__(self, key, model_name, base_url="https://api.siliconflow.cn/v1/rerank"):
+ def __init__(self, key, model_name, base_url="https://api.siliconflow.cn/v1/rerank", max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
normalized_base_url = (base_url or "").strip()
if not normalized_base_url:
normalized_base_url = "https://api.siliconflow.cn/v1/rerank"
@@ -585,7 +589,8 @@ class SILICONFLOWRerank(Base):
class BaiduYiyanRerank(Base):
_FACTORY_NAME = "BaiduYiyan"
- def __init__(self, key, model_name, base_url=None):
+ def __init__(self, key, model_name, base_url=None, max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
from qianfan.resources import Reranker
try:
@@ -620,7 +625,8 @@ class BaiduYiyanRerank(Base):
class VoyageRerank(Base):
_FACTORY_NAME = "Voyage AI"
- def __init__(self, key, model_name, base_url=None):
+ def __init__(self, key, model_name, base_url=None, max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
import voyageai
self.client = voyageai.Client(api_key=key, timeout=30.0)
@@ -641,7 +647,8 @@ class VoyageRerank(Base):
class QWenRerank(Base):
_FACTORY_NAME = "Tongyi-Qianwen"
- def __init__(self, key, model_name="gte-rerank-v2", **kwargs):
+ def __init__(self, key, model_name="gte-rerank-v2", max_token=MAX_RERANK_TOKEN, **kwargs):
+ super().__init__(key, model_name, max_token)
self.api_key = key
self.model_name = model_name if model_name else "gte-rerank-v2"
# Remove invalid global timeout, use official SDK per-request timeout parameter
@@ -696,7 +703,7 @@ class HuggingfaceRerank(Base):
try:
# Fix: Add request timeout
res = requests.post(
- endpoint, headers={"Content-Type": "application/json"}, json={"query": query, "texts": texts[i : i + batch_size], "raw_scores": False, "truncate": True}, timeout=30
+ endpoint, headers={"Content-Type": "application/json"}, json={"query": query, "texts": texts[i : i + batch_size], "raw_scores": False, "truncate": False}, timeout=30
)
res.raise_for_status()
for o in res.json():
@@ -708,7 +715,8 @@ class HuggingfaceRerank(Base):
raise exc
return np.array(scores)
- def __init__(self, key, model_name="BAAI/bge-reranker-v2-m3", base_url="http://127.0.0.1"):
+ def __init__(self, key, model_name="BAAI/bge-reranker-v2-m3", base_url="http://127.0.0.1", max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
self.model_name = model_name.split("___")[0]
self.base_url = base_url
@@ -724,7 +732,8 @@ class GPUStackRerank(Base):
_FACTORY_NAME = "GPUStack"
- def __init__(self, key, model_name, base_url):
+ def __init__(self, key, model_name, base_url, max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
if not base_url:
raise ValueError("url cannot be None")
@@ -766,32 +775,32 @@ class GPUStackRerank(Base):
class NovitaRerank(JinaRerank):
_FACTORY_NAME = "NovitaAI"
- def __init__(self, key, model_name, base_url="https://api.novita.ai/v3/openai/rerank"):
+ def __init__(self, key, model_name, base_url="https://api.novita.ai/v3/openai/rerank", max_token=MAX_RERANK_TOKEN):
if not base_url:
base_url = "https://api.novita.ai/v3/openai/rerank"
- super().__init__(key, model_name, base_url)
+ super().__init__(key, model_name, base_url, max_token)
class GiteeRerank(JinaRerank):
_FACTORY_NAME = "GiteeAI"
- def __init__(self, key, model_name, base_url="https://api.moark.com/v1/rerank"):
+ def __init__(self, key, model_name, base_url="https://api.moark.com/v1/rerank", max_token=MAX_RERANK_TOKEN):
endpoint = (base_url or "https://api.moark.com/v1/rerank").rstrip("/")
if endpoint.endswith("/v1"):
endpoint += "/rerank"
- super().__init__(key, model_name, base_url=endpoint)
+ super().__init__(key, model_name, base_url=endpoint, max_token=max_token)
class Ai302Rerank(Base):
_FACTORY_NAME = "302.AI"
- def __init__(self, key, model_name, base_url="https://api.302.ai/v1/rerank"):
+ def __init__(self, key, model_name, base_url="https://api.302.ai/v1/rerank", max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
self.base_url = base_url or "https://api.302.ai/v1/rerank"
self.headers = {"Content-Type": "application/json", "Authorization": f"Bearer {key}"}
self.model_name = model_name
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
- texts = [truncate(t, 500) for t in texts]
data = {"model": self.model_name, "query": query, "documents": texts, "top_n": len(texts)}
response = requests.post(self.base_url, headers=self.headers, json=data, timeout=30)
response.raise_for_status()
@@ -808,26 +817,27 @@ class Ai302Rerank(Base):
class JiekouAIRerank(JinaRerank):
_FACTORY_NAME = "Jiekou.AI"
- def __init__(self, key, model_name, base_url="https://api.jiekou.ai/openai/v1/rerank"):
+ def __init__(self, key, model_name, base_url="https://api.jiekou.ai/openai/v1/rerank", max_token=MAX_RERANK_TOKEN):
if not base_url:
base_url = "https://api.jiekou.ai/openai/v1/rerank"
- super().__init__(key, model_name, base_url)
+ super().__init__(key, model_name, base_url, max_token)
class FuturMixRerank(OpenAI_APIRerank):
_FACTORY_NAME = "FuturMix"
- def __init__(self, key, model_name, base_url="https://futurmix.ai/v1/rerank"):
+ def __init__(self, key, model_name, base_url="https://futurmix.ai/v1/rerank", max_token=MAX_RERANK_TOKEN):
if not base_url:
base_url = "https://futurmix.ai/v1/rerank"
- super().__init__(key, model_name, base_url)
+ super().__init__(key, model_name, base_url, max_token)
logging.info("[FuturMix] Rerank initialized with model %s", model_name)
class RAGconRerank(Base):
_FACTORY_NAME = "RAGcon"
- def __init__(self, key, model_name, base_url=None, **kwargs):
+ def __init__(self, key, model_name, base_url=None, max_token=MAX_RERANK_TOKEN, **kwargs):
+ super().__init__(key, model_name, max_token)
if not base_url:
base_url = "https://connect.ragcon.com/v1"
@@ -838,7 +848,6 @@ class RAGconRerank(Base):
self.model_name = model_name
def _compute_rank(self, query: str, texts: List) -> Tuple[np.ndarray, int]:
- texts = [truncate(t, 500) for t in texts]
data = {
"model": self.model_name,
"query": query,
@@ -861,7 +870,8 @@ class RAGconRerank(Base):
class NewAPIRerank(Base):
_FACTORY_NAME = "New API"
- def __init__(self, key, model_name, base_url):
+ def __init__(self, key, model_name, base_url, max_token=MAX_RERANK_TOKEN):
+ super().__init__(key, model_name, max_token)
normalized_base_url = (base_url or "").strip()
if "/rerank" in normalized_base_url:
self.base_url = normalized_base_url.rstrip("/")
@@ -874,7 +884,6 @@ class NewAPIRerank(Base):
self.model_name = model_name.split("___")[0]
def _compute_rank(self, query: str, texts: list):
- texts = [truncate(t, 500) for t in texts]
data = {
"model": self.model_name,
"query": query,
diff --git a/test/unit_test/rag/llm/test_bedrock_rerank.py b/test/unit_test/rag/llm/test_bedrock_rerank.py
index 68a8430baf..3a7b76c1de 100644
--- a/test/unit_test/rag/llm/test_bedrock_rerank.py
+++ b/test/unit_test/rag/llm/test_bedrock_rerank.py
@@ -18,7 +18,7 @@
``BedrockRerank`` talks to the ``bedrock-agent-runtime`` Rerank API via boto3.
These tests patch ``boto3.client`` so no AWS call is made, and verify the
-score-by-index mapping, per-model document truncation and key/ARN handling.
+score-by-index mapping, document token limiting and key/ARN handling.
"""
import json
@@ -69,19 +69,13 @@ def test_model_arn_is_built_from_region_and_name():
assert mdl.model_arn == "arn:aws:bedrock:eu-central-1::foundation-model/cohere.rerank-v3-5:0"
-def test_doc_window_depends_on_model():
- cohere, _ = _make(model_name="cohere.rerank-v3-5:0")
- amazon, _ = _make(model_name="amazon.rerank-v1:0")
- assert cohere.doc_max_tokens == 2048
- assert amazon.doc_max_tokens == 8192
-
-
-def test_documents_are_truncated_before_send():
- mdl, client = _make(model_name="cohere.rerank-v3-5:0") # cap 2048
+def test_documents_are_truncated_to_configured_max_tokens_before_send():
+ mdl, client = _make(model_name="cohere.rerank-v3-5:0")
+ mdl.max_token = 2048
client.rerank.return_value = _rerank_response([(0, 0.5)])
mdl.similarity("q", ["mot " * 5000]) # > 2048 tokens
sent = client.rerank.call_args.kwargs["sources"][0]["inlineDocumentSource"]["textDocument"]["text"]
- assert num_tokens_from_string(sent) <= 2048
+ assert num_tokens_from_string("q") + num_tokens_from_string(sent) <= 2048
def test_number_of_results_covers_all_documents():
@@ -135,25 +129,7 @@ def test_empty_input_short_circuits_without_calling_bedrock(query, texts):
client.rerank.assert_not_called()
-def test_document_is_capped_at_32000_chars():
- # RerankTextDocument.text is hard-capped at 32,000 chars; a longer doc would
- # otherwise be rejected by the API. Amazon's 8k-token window can exceed it.
- mdl, client = _make(model_name="amazon.rerank-v1:0")
- client.rerank.return_value = _rerank_response([(0, 0.5)])
- mdl.similarity("q", ["word " * 30000]) # 150k chars -> ~41k after token truncation
- sent = client.rerank.call_args.kwargs["sources"][0]["inlineDocumentSource"]["textDocument"]["text"]
- assert len(sent) == 32000
-
-
-def test_query_is_capped_at_32000_chars():
- mdl, client = _make(model_name="amazon.rerank-v1:0")
- client.rerank.return_value = _rerank_response([(0, 0.5)])
- mdl.similarity("q " * 30000, ["doc"]) # oversize query
- sent = client.rerank.call_args.kwargs["queries"][0]["textQuery"]["text"]
- assert len(sent) == 32000
-
-
-def test_short_document_is_not_char_capped():
+def test_document_is_sent_unchanged_within_token_limit():
mdl, client = _make(model_name="amazon.rerank-v1:0")
client.rerank.return_value = _rerank_response([(0, 0.5)])
mdl.similarity("q", ["short document"])
diff --git a/test/unit_test/rag/llm/test_rerank_normalization.py b/test/unit_test/rag/llm/test_rerank_normalization.py
index 26a1649a12..b2261a8a5e 100644
--- a/test/unit_test/rag/llm/test_rerank_normalization.py
+++ b/test/unit_test/rag/llm/test_rerank_normalization.py
@@ -49,6 +49,7 @@ class _RawRerank(Base):
"""Minimal provider that emits arbitrary raw scores via ``_compute_rank``."""
def __init__(self, raw):
+ super().__init__(None, None)
self._raw = np.asarray(raw, dtype=float)
def _compute_rank(self, query, texts):