Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor: use model tokenizer for rerank (#20422)

buua436c8a00c6f6de0
diff --git a/conf/all_models.json b/conf/all_models.json
index cdc0597d2d..ddab90a479 100644
--- a/conf/all_models.json
+++ b/conf/all_models.json
@@ -1062,6 +1062,7 @@
"model_types": [
"rerank"
],
+ "tokenizer": "xlmr-spm",
"context_length": 1024,
"max_output": 1024
},
@@ -1096,6 +1097,7 @@
"model_types": [
"embedding"
],
+ "tokenizer": "bert-wordpiece",
"max_batch_size": 2048
},
{
@@ -2111,6 +2113,7 @@
"model_types": [
"embedding"
],
+ "tokenizer": "bert-wordpiece",
"max_batch_size": 2048
},
{
@@ -13589,6 +13592,7 @@
"model_types": [
"rerank"
],
+ "tokenizer": "qwen-bpe",
"context_length": 8192,
"max_output": 8192
},
@@ -13600,6 +13604,7 @@
"model_types": [
"rerank"
],
+ "tokenizer": "qwen-bpe",
"context_length": 8192,
"max_output": 8192
},
@@ -13611,6 +13616,7 @@
"model_types": [
"rerank"
],
+ "tokenizer": "qwen-bpe",
"context_length": 8192,
"max_output": 8192
},
@@ -13656,6 +13662,7 @@
},
{
"name": "jinaai/jina-colbert-v2",
+ "tokenizer": "xlmr-spm",
"alias": [
"jina-colbert-v2"
],
@@ -13672,6 +13679,7 @@
},
{
"name": "jinaai/jina-colbert-v2-64",
+ "tokenizer": "xlmr-spm",
"alias": [
"jina-colbert-v2-64"
],
@@ -13796,6 +13804,7 @@
"model_types": [
"rerank"
],
+ "tokenizer": "xlmr-spm",
"context_length": 8192,
"max_output": 8192
},
@@ -28411,6 +28420,7 @@
"model_types": [
"rerank"
],
+ "tokenizer": "qwen-bpe",
"context_length": 32768,
"max_output": 32768
},
@@ -28422,6 +28432,7 @@
"model_types": [
"rerank"
],
+ "tokenizer": "qwen-bpe",
"context_length": 32768,
"max_output": 32768
},
@@ -28433,6 +28444,7 @@
"model_types": [
"rerank"
],
+ "tokenizer": "qwen-bpe",
"context_length": 32768,
"max_output": 32768
},
@@ -33945,6 +33957,7 @@
},
{
"name": "sentence-transformers/all-MiniLM-L12-v2",
+ "tokenizer": "bert-wordpiece",
"alias": [
"all-MiniLM-L12-v2"
],
@@ -33958,6 +33971,7 @@
},
{
"name": "sentence-transformers/all-MiniLM-L6-v2",
+ "tokenizer": "bert-wordpiece",
"alias": [
"all-MiniLM-L6-v2"
],
@@ -33984,6 +33998,7 @@
},
{
"name": "sentence-transformers/paraphrase-MiniLM-L6-v2",
+ "tokenizer": "bert-wordpiece",
"alias": [
"paraphrase-MiniLM-L6-v2"
],
@@ -34085,6 +34100,7 @@
},
{
"name": "shibing624/text2vec-base-chinese",
+ "tokenizer": "bert-wordpiece",
"alias": [
"text2vec-base-chinese"
],
diff --git a/internal/entity/models/model.go b/internal/entity/models/model.go
index 861b54b4bd..ef8a414595 100644
--- a/internal/entity/models/model.go
+++ b/internal/entity/models/model.go
@@ -494,7 +494,7 @@ func GetEmbeddingBatchSize(modelName string) int {
return DefaultEmbeddingBatchSize
}
-// GetEmbeddingTokenizer returns the tokenizer id declared for the named model in
+// GetModelTokenizer returns the tokenizer id declared for the named model in
// the provider catalog (conf/all_models.json "tokenizer"), or "" when the model
// is unknown or declares none. The TOKENIZER_EMBEDDING_TOKENIZER env var overrides
// everything, so an operator can pin a family without editing the catalog.
@@ -503,7 +503,7 @@ func GetEmbeddingBatchSize(modelName string) int {
// is missing may fall back to a calibrated cl100k count is the CALLER's decision - the
// ingest embedder refuses (internal/ingestion/task/embedder.go), while a model that
// declares no tokenizer at all is calibrated by design.
-func GetEmbeddingTokenizer(modelName string) string {
+func GetModelTokenizer(modelName string) string {
if v := strings.TrimSpace(os.Getenv("TOKENIZER_EMBEDDING_TOKENIZER")); v != "" {
return v
}
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index b127ecd138..310f2d8524 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -331,7 +331,7 @@ func (m *EmbeddingModel) ResolveTokenizerID() string {
if m.ModelName != nil {
name = *m.ModelName
}
- return GetEmbeddingTokenizer(name)
+ return GetModelTokenizer(name)
}
// QuotaKey names the deployment this embedding model counts against: endpoint,
@@ -370,6 +370,9 @@ type RerankModel struct {
ModelName *string
APIConfig *APIConfig
MaxTokens int
+
+ limiter tokenizer.Limiter
+ limiterErr error
}
// NewRerankModel creates a new RerankModel
@@ -377,20 +380,45 @@ func NewRerankModel(driver ModelDriver, modelName *string, apiConfig *APIConfig,
if maxTokens <= 0 {
maxTokens = defaultMaxRerankTokens
}
+ tokenizerID := ""
+ if modelName != nil {
+ tokenizerID = GetModelTokenizer(*modelName)
+ }
return &RerankModel{
ModelDriver: driver,
ModelName: modelName,
APIConfig: apiConfig,
MaxTokens: maxTokens,
+ limiter: tokenizer.LimiterFor(tokenizerID, "", tokenizer.DefaultCalibration()),
+ limiterErr: tokenizer.RefuseUnavailableCounter(tokenizerID, "rerank"),
+ }
+}
+
+// ResolveTokenizerID returns the tokenizer family declared for this reranker, or
+// an empty string when the model catalog does not provide one.
+func (r *RerankModel) ResolveTokenizerID() string {
+ if r == nil || r.ModelName == nil {
+ return ""
}
+ return GetModelTokenizer(*r.ModelName)
}
-// Rerank calculates similarity between query and texts
+// Rerank calculates similarity between query and texts. Rerank input is measured
+// and truncated with the model's declared tokenizer, just like embedding input.
func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, apiConfig *APIConfig, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error) {
+ if r == nil || r.ModelDriver == nil {
+ return nil, errors.New("rerank model: driver is nil")
+ }
maxTokens := r.MaxTokens
if maxTokens <= 0 {
maxTokens = defaultMaxRerankTokens
}
+ if r.limiterErr != nil {
+ return nil, r.limiterErr
+ }
+ limiter := r.limiter
+ counter := limiter.Counter()
+ effectiveMaxTokens := limiter.Limit(maxTokens)
mode := strings.ToLower(strings.TrimSpace(common.GetEnv(common.EnvRerankTokenLimitMode)))
if mode == "" {
mode = "truncate"
@@ -398,20 +426,21 @@ func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, apiConf
if mode != "truncate" && mode != "passthrough" && mode != "raise_error" {
return nil, fmt.Errorf("%w: invalid %s %q; expected %q, %q, or %q", ErrRerankTokenLimitPolicy, common.EnvRerankTokenLimitMode, mode, "truncate", "passthrough", "raise_error")
}
- if mode != "passthrough" && request.Query != "" && len(request.Documents) > 0 {
- queryTokens := tokenizer.NumTokensFromString(request.Query)
+ if mode != "passthrough" && len(request.Documents) > 0 {
+ queryTokens := counter.Count(request.Query)
if mode == "truncate" {
- documentTokens := max(maxTokens-queryTokens, 0)
+ documentBudget := max(effectiveMaxTokens-queryTokens, 0)
+ documentTokens := tokenizer.OverLimitLadder(documentBudget)[0]
documents := make([]string, len(request.Documents))
for i, document := range request.Documents {
- documents[i] = tokenizer.TrimContentToTokenLimit(document, documentTokens)
+ documents[i] = counter.TrimToLimit(document, documentTokens)
}
request.Documents = documents
} else {
for i, document := range request.Documents {
- inputTokens := queryTokens + tokenizer.NumTokensFromString(document)
- if inputTokens > maxTokens {
- return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, maxTokens)
+ inputTokens := queryTokens + counter.Count(document)
+ if inputTokens > effectiveMaxTokens {
+ return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, effectiveMaxTokens)
}
}
}
diff --git a/internal/service/model_service.go b/internal/service/model_service.go
index 803dd03d95..d8cd576212 100644
--- a/internal/service/model_service.go
+++ b/internal/service/model_service.go
@@ -974,7 +974,8 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, pr
Query: "test",
Documents: []string{"test"},
}
- _, err = driver.Rerank(ctx, &modelName, rerankRequest, apiConfig, &modelModule.RerankConfig{}, nil)
+ rerankModel := modelModule.NewRerankModel(driver, &modelName, apiConfig, 0)
+ _, err = rerankModel.Rerank(ctx, rerankRequest, apiConfig, &modelModule.RerankConfig{}, nil)
case "tts":
content := "hello"
ttsModel := modelModule.NewTTSModel(driver, &modelName, apiConfig)
diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.go
index 3fde4ebcf9..769554fcb9 100644
--- a/internal/service/nlp/reranker_question_tks_test.go
+++ b/internal/service/nlp/reranker_question_tks_test.go
@@ -47,7 +47,7 @@ func rerankOneChunk(t *testing.T, chunk map[string]interface{}) string {
RerankByModel(
context.Background(),
- &models.RerankModel{ModelDriver: driver},
+ models.NewRerankModel(driver, nil, nil, 0),
[]map[string]interface{}{chunk},
ids,
field,
diff --git a/internal/tokenizer/embedlimit.go b/internal/tokenizer/embedlimit.go
index 4398b623f1..b6990df1ba 100644
--- a/internal/tokenizer/embedlimit.go
+++ b/internal/tokenizer/embedlimit.go
@@ -663,7 +663,7 @@ func RefuseUnavailableCounter(tokenizerID, calibrationKey string) error {
return nil
}
return fmt.Errorf(
- "embedding tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_go_deps.py`): refusing to count with the calibrated estimate",
+ "model tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_go_deps.py`): refusing to count with the calibrated estimate",
tokenizerID, calibrationKey)
}