RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor: use model tokenizer for rerank (#20422)
buua436c8a00c6f6de0
diff --git a/conf/all_models.json b/conf/all_models.jsonindex cdc0597d2d..ddab90a479 100644--- a/conf/all_models.json+++ b/conf/all_models.json@@ -1062,6 +1062,7 @@"model_types": ["rerank"],+ "tokenizer": "xlmr-spm","context_length": 1024,"max_output": 1024},@@ -1096,6 +1097,7 @@"model_types": ["embedding"],+ "tokenizer": "bert-wordpiece","max_batch_size": 2048},{@@ -2111,6 +2113,7 @@"model_types": ["embedding"],+ "tokenizer": "bert-wordpiece","max_batch_size": 2048},{@@ -13589,6 +13592,7 @@"model_types": ["rerank"],+ "tokenizer": "qwen-bpe","context_length": 8192,"max_output": 8192},@@ -13600,6 +13604,7 @@"model_types": ["rerank"],+ "tokenizer": "qwen-bpe","context_length": 8192,"max_output": 8192},@@ -13611,6 +13616,7 @@"model_types": ["rerank"],+ "tokenizer": "qwen-bpe","context_length": 8192,"max_output": 8192},@@ -13656,6 +13662,7 @@},{"name": "jinaai/jina-colbert-v2",+ "tokenizer": "xlmr-spm","alias": ["jina-colbert-v2"],@@ -13672,6 +13679,7 @@},{"name": "jinaai/jina-colbert-v2-64",+ "tokenizer": "xlmr-spm","alias": ["jina-colbert-v2-64"],@@ -13796,6 +13804,7 @@"model_types": ["rerank"],+ "tokenizer": "xlmr-spm","context_length": 8192,"max_output": 8192},@@ -28411,6 +28420,7 @@"model_types": ["rerank"],+ "tokenizer": "qwen-bpe","context_length": 32768,"max_output": 32768},@@ -28422,6 +28432,7 @@"model_types": ["rerank"],+ "tokenizer": "qwen-bpe","context_length": 32768,"max_output": 32768},@@ -28433,6 +28444,7 @@"model_types": ["rerank"],+ "tokenizer": "qwen-bpe","context_length": 32768,"max_output": 32768},@@ -33945,6 +33957,7 @@},{"name": "sentence-transformers/all-MiniLM-L12-v2",+ "tokenizer": "bert-wordpiece","alias": ["all-MiniLM-L12-v2"],@@ -33958,6 +33971,7 @@},{"name": "sentence-transformers/all-MiniLM-L6-v2",+ "tokenizer": "bert-wordpiece","alias": ["all-MiniLM-L6-v2"],@@ -33984,6 +33998,7 @@},{"name": "sentence-transformers/paraphrase-MiniLM-L6-v2",+ "tokenizer": "bert-wordpiece","alias": ["paraphrase-MiniLM-L6-v2"],@@ -34085,6 +34100,7 @@},{"name": "shibing624/text2vec-base-chinese",+ "tokenizer": "bert-wordpiece","alias": ["text2vec-base-chinese"],diff --git a/internal/entity/models/model.go b/internal/entity/models/model.goindex 861b54b4bd..ef8a414595 100644--- a/internal/entity/models/model.go+++ b/internal/entity/models/model.go@@ -494,7 +494,7 @@ func GetEmbeddingBatchSize(modelName string) int {return DefaultEmbeddingBatchSize}-// GetEmbeddingTokenizer returns the tokenizer id declared for the named model in+// GetModelTokenizer returns the tokenizer id declared for the named model in// the provider catalog (conf/all_models.json "tokenizer"), or "" when the model// is unknown or declares none. The TOKENIZER_EMBEDDING_TOKENIZER env var overrides// everything, so an operator can pin a family without editing the catalog.@@ -503,7 +503,7 @@ func GetEmbeddingBatchSize(modelName string) int {// is missing may fall back to a calibrated cl100k count is the CALLER's decision - the// ingest embedder refuses (internal/ingestion/task/embedder.go), while a model that// declares no tokenizer at all is calibrated by design.-func GetEmbeddingTokenizer(modelName string) string {+func GetModelTokenizer(modelName string) string {if v := strings.TrimSpace(os.Getenv("TOKENIZER_EMBEDDING_TOKENIZER")); v != "" {return v}diff --git a/internal/entity/models/types.go b/internal/entity/models/types.goindex b127ecd138..310f2d8524 100644--- a/internal/entity/models/types.go+++ b/internal/entity/models/types.go@@ -331,7 +331,7 @@ func (m *EmbeddingModel) ResolveTokenizerID() string {if m.ModelName != nil {name = *m.ModelName}- return GetEmbeddingTokenizer(name)+ return GetModelTokenizer(name)}// QuotaKey names the deployment this embedding model counts against: endpoint,@@ -370,6 +370,9 @@ type RerankModel struct {ModelName *stringAPIConfig *APIConfigMaxTokens int++ limiter tokenizer.Limiter+ limiterErr error}// NewRerankModel creates a new RerankModel@@ -377,20 +380,45 @@ func NewRerankModel(driver ModelDriver, modelName *string, apiConfig *APIConfig,if maxTokens <= 0 {maxTokens = defaultMaxRerankTokens}+ tokenizerID := ""+ if modelName != nil {+ tokenizerID = GetModelTokenizer(*modelName)+ }return &RerankModel{ModelDriver: driver,ModelName: modelName,APIConfig: apiConfig,MaxTokens: maxTokens,+ limiter: tokenizer.LimiterFor(tokenizerID, "", tokenizer.DefaultCalibration()),+ limiterErr: tokenizer.RefuseUnavailableCounter(tokenizerID, "rerank"),+ }+}++// ResolveTokenizerID returns the tokenizer family declared for this reranker, or+// an empty string when the model catalog does not provide one.+func (r *RerankModel) ResolveTokenizerID() string {+ if r == nil || r.ModelName == nil {+ return ""}+ return GetModelTokenizer(*r.ModelName)}-// Rerank calculates similarity between query and texts+// Rerank calculates similarity between query and texts. Rerank input is measured+// and truncated with the model's declared tokenizer, just like embedding input.func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, apiConfig *APIConfig, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error) {+ if r == nil || r.ModelDriver == nil {+ return nil, errors.New("rerank model: driver is nil")+ }maxTokens := r.MaxTokensif maxTokens <= 0 {maxTokens = defaultMaxRerankTokens}+ if r.limiterErr != nil {+ return nil, r.limiterErr+ }+ limiter := r.limiter+ counter := limiter.Counter()+ effectiveMaxTokens := limiter.Limit(maxTokens)mode := strings.ToLower(strings.TrimSpace(common.GetEnv(common.EnvRerankTokenLimitMode)))if mode == "" {mode = "truncate"@@ -398,20 +426,21 @@ func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, apiConfif mode != "truncate" && mode != "passthrough" && mode != "raise_error" {return nil, fmt.Errorf("%w: invalid %s %q; expected %q, %q, or %q", ErrRerankTokenLimitPolicy, common.EnvRerankTokenLimitMode, mode, "truncate", "passthrough", "raise_error")}- if mode != "passthrough" && request.Query != "" && len(request.Documents) > 0 {- queryTokens := tokenizer.NumTokensFromString(request.Query)+ if mode != "passthrough" && len(request.Documents) > 0 {+ queryTokens := counter.Count(request.Query)if mode == "truncate" {- documentTokens := max(maxTokens-queryTokens, 0)+ documentBudget := max(effectiveMaxTokens-queryTokens, 0)+ documentTokens := tokenizer.OverLimitLadder(documentBudget)[0]documents := make([]string, len(request.Documents))for i, document := range request.Documents {- documents[i] = tokenizer.TrimContentToTokenLimit(document, documentTokens)+ documents[i] = counter.TrimToLimit(document, documentTokens)}request.Documents = documents} else {for i, document := range request.Documents {- inputTokens := queryTokens + tokenizer.NumTokensFromString(document)- if inputTokens > maxTokens {- return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, maxTokens)+ inputTokens := queryTokens + counter.Count(document)+ if inputTokens > effectiveMaxTokens {+ return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, effectiveMaxTokens)}}}diff --git a/internal/service/model_service.go b/internal/service/model_service.goindex 803dd03d95..d8cd576212 100644--- a/internal/service/model_service.go+++ b/internal/service/model_service.go@@ -974,7 +974,8 @@ func verifyProviderModel(ctx context.Context, driver modelModule.ModelDriver, prQuery: "test",Documents: []string{"test"},}- _, err = driver.Rerank(ctx, &modelName, rerankRequest, apiConfig, &modelModule.RerankConfig{}, nil)+ rerankModel := modelModule.NewRerankModel(driver, &modelName, apiConfig, 0)+ _, err = rerankModel.Rerank(ctx, rerankRequest, apiConfig, &modelModule.RerankConfig{}, nil)case "tts":content := "hello"ttsModel := modelModule.NewTTSModel(driver, &modelName, apiConfig)diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.goindex 3fde4ebcf9..769554fcb9 100644--- a/internal/service/nlp/reranker_question_tks_test.go+++ b/internal/service/nlp/reranker_question_tks_test.go@@ -47,7 +47,7 @@ func rerankOneChunk(t *testing.T, chunk map[string]interface{}) string {RerankByModel(context.Background(),- &models.RerankModel{ModelDriver: driver},+ models.NewRerankModel(driver, nil, nil, 0),[]map[string]interface{}{chunk},ids,field,diff --git a/internal/tokenizer/embedlimit.go b/internal/tokenizer/embedlimit.goindex 4398b623f1..b6990df1ba 100644--- a/internal/tokenizer/embedlimit.go+++ b/internal/tokenizer/embedlimit.go@@ -663,7 +663,7 @@ func RefuseUnavailableCounter(tokenizerID, calibrationKey string) error {return nil}return fmt.Errorf(- "embedding tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_go_deps.py`): refusing to count with the calibrated estimate",+ "model tokenizer %q is declared for %s but its asset is unavailable (check that ragflow_deps/huggingface.co is present; run `uv run ragflow_deps/download_go_deps.py`): refusing to count with the calibrated estimate",tokenizerID, calibrationKey)}