Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor: Follow #19404 to add RERANK_TOKEN_LIMIT_MODE=truncate|passthrough|raise_error (#19421)

Wang Qi2284c8e1abcd
diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.go
index 437a12dd61..e8c1e09bba 100644
--- a/internal/agent/tool/retrieval_nlp.go
+++ b/internal/agent/tool/retrieval_nlp.go
@@ -518,15 +518,16 @@ func (a *NLPRetrievalAdapter) resolveRerankModel(
driver modelModule.ModelDriver
modelName string
apiConfig *modelModule.APIConfig
+ maxTokens int
err error
)
- driver, modelName, apiConfig, _, err = a.modelResolver.ResolveModelConfig(
+ driver, modelName, apiConfig, maxTokens, err = a.modelResolver.ResolveModelConfig(
ctx, tenantID, entity.ModelTypeRerank, req.RerankID,
)
if err != nil {
return nil, fmt.Errorf("retrieval: resolve rerank model: %w", err)
}
- return modelModule.NewRerankModel(driver, &modelName, apiConfig), nil
+ return modelModule.NewRerankModel(driver, &modelName, apiConfig, maxTokens), nil
}
// translateChunk converts one nlp chunk map into a RetrievalChunk.
diff --git a/internal/common/environments.go b/internal/common/environments.go
index ef807a8935..3b4ed59c97 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -34,6 +34,7 @@ func GetEnvSmall(key string) string {
const (
EnvTensorrtDLAServer = "TENSORRT_DLA_SVR"
EnvRAGFlowTTSCacheTTLSeconds = "RAGFLOW_TTS_CACHE_TTL_SECONDS"
+ EnvRerankTokenLimitMode = "RERANK_TOKEN_LIMIT_MODE"
EnvComponentExecTimeout = "COMPONENT_EXEC_TIMEOUT"
EnvDocEngine = "DOC_ENGINE"
EnvMaxFileNumPerUser = "MAX_FILE_NUM_PER_USER"
diff --git a/internal/entity/models/nvidia.go b/internal/entity/models/nvidia.go
index d37345c4c8..85c35af18b 100644
--- a/internal/entity/models/nvidia.go
+++ b/internal/entity/models/nvidia.go
@@ -239,13 +239,11 @@ func (n *NvidiaModel) Embed(ctx context.Context, modelName *string, request Embe
// nvidiaRerankRequest mirrors the NIM /ranking request shape:
// query is an object with a "text" field, passages is an array of
-// objects each with a "text" field. truncate=END matches the Python
-// NvidiaRerank reference at rag/llm/rerank_model.py.
+// objects each with a "text" field.
type nvidiaRerankRequest struct {
Model string `json:"model"`
Query nvidiaRerankText `json:"query"`
Passages []nvidiaRerankText `json:"passages"`
- Truncate string `json:"truncate,omitempty"`
TopN int `json:"top_n"`
}
@@ -304,7 +302,6 @@ func (n *NvidiaModel) Rerank(ctx context.Context, modelName *string, request Rer
Model: *modelName,
Query: nvidiaRerankText{Text: query},
Passages: passages,
- Truncate: "END",
TopN: topN,
}
diff --git a/internal/entity/models/nvidia_rerank_test.go b/internal/entity/models/nvidia_rerank_test.go
index 0a4b055406..7d9237c14e 100644
--- a/internal/entity/models/nvidia_rerank_test.go
+++ b/internal/entity/models/nvidia_rerank_test.go
@@ -67,8 +67,8 @@ func TestNvidiaRerankHappyPath(t *testing.T) {
t.Errorf("expected 3 passages, got %v", body["passages"])
return
}
- if body["truncate"] != "END" {
- t.Errorf("expected truncate=END, got %v", body["truncate"])
+ if _, ok := body["truncate"]; ok {
+ t.Errorf("expected provider-side truncation to be disabled, got %v", body["truncate"])
}
if body["top_n"] != float64(3) {
t.Errorf("expected top_n=3 (matching len(documents)), got %v", body["top_n"])
diff --git a/internal/entity/models/rerank_model_test.go b/internal/entity/models/rerank_model_test.go
new file mode 100644
index 0000000000..4a6ca3ef40
--- /dev/null
+++ b/internal/entity/models/rerank_model_test.go
@@ -0,0 +1,91 @@
+package models
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/tokenizer"
+)
+
+type captureRerankDriver struct {
+ ModelDriver
+ request RerankRequest
+ calls int
+}
+
+func (d *captureRerankDriver) Rerank(_ context.Context, _ *string, request RerankRequest, _ *APIConfig, _ *RerankConfig, _ *common.ModelUsage) (*RerankResponse, error) {
+ d.request = request
+ d.calls++
+ return &RerankResponse{}, nil
+}
+
+func TestRerankModelTokenLimitModes(t *testing.T) {
+ query := "question"
+ document := strings.Repeat("token ", 20)
+ queryTokens := tokenizer.NumTokensFromString(query)
+
+ t.Run("truncate", func(t *testing.T) {
+ t.Setenv(common.EnvRerankTokenLimitMode, "")
+ driver := &captureRerankDriver{}
+ model := NewRerankModel(driver, nil, nil, queryTokens+4)
+ _, err := model.Rerank(t.Context(), RerankRequest{Query: query, Documents: []string{document}}, nil, nil, nil)
+ if err != nil {
+ t.Fatalf("Rerank() error = %v", err)
+ }
+ if got := queryTokens + tokenizer.NumTokensFromString(driver.request.Documents[0]); got > model.MaxTokens {
+ t.Fatalf("rerank input tokens = %d, want at most %d", got, model.MaxTokens)
+ }
+ if driver.request.Documents[0] == document {
+ t.Fatal("document was not truncated")
+ }
+ })
+
+ t.Run("passthrough", func(t *testing.T) {
+ t.Setenv(common.EnvRerankTokenLimitMode, " PASSTHROUGH ")
+ driver := &captureRerankDriver{}
+ model := NewRerankModel(driver, nil, nil, 1)
+ _, err := model.Rerank(t.Context(), RerankRequest{Query: query, Documents: []string{document}}, nil, nil, nil)
+ if err != nil {
+ t.Fatalf("Rerank() error = %v", err)
+ }
+ if driver.request.Documents[0] != document {
+ t.Fatalf("document = %q, want unchanged input", driver.request.Documents[0])
+ }
+ })
+
+ t.Run("raise_error", func(t *testing.T) {
+ t.Setenv(common.EnvRerankTokenLimitMode, "raise_error")
+ driver := &captureRerankDriver{}
+ model := NewRerankModel(driver, nil, nil, queryTokens)
+ _, err := model.Rerank(t.Context(), RerankRequest{Query: query, Documents: []string{"short"}}, nil, nil, nil)
+ if !errors.Is(err, ErrRerankTokenLimitPolicy) || !strings.Contains(err.Error(), "document index 0") {
+ t.Fatalf("Rerank() error = %v, want token-limit error for document 0", err)
+ }
+ if driver.calls != 0 {
+ t.Fatalf("driver calls = %d, want 0", driver.calls)
+ }
+ })
+
+ t.Run("invalid", func(t *testing.T) {
+ t.Setenv(common.EnvRerankTokenLimitMode, "invalid")
+ driver := &captureRerankDriver{}
+ model := NewRerankModel(driver, nil, nil, 10)
+ _, err := model.Rerank(t.Context(), RerankRequest{Query: query, Documents: []string{"short"}}, nil, nil, nil)
+ if !errors.Is(err, ErrRerankTokenLimitPolicy) || !strings.Contains(err.Error(), common.EnvRerankTokenLimitMode) {
+ t.Fatalf("Rerank() error = %v, want invalid-mode error", err)
+ }
+ if driver.calls != 0 {
+ t.Fatalf("driver calls = %d, want 0", driver.calls)
+ }
+ })
+}
+
+func TestNewRerankModelDefaultsMaxTokens(t *testing.T) {
+ model := NewRerankModel(&captureRerankDriver{}, nil, nil, 0)
+ if model.MaxTokens != defaultMaxRerankTokens {
+ t.Fatalf("MaxTokens = %d, want %d", model.MaxTokens, defaultMaxRerankTokens)
+ }
+}
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index ebae5789a3..a21c3d516e 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -3,9 +3,19 @@ package models
import (
"context"
"encoding/json"
+ "errors"
+ "fmt"
+ "strings"
+
"ragflow/internal/common"
+ "ragflow/internal/tokenizer"
)
+const defaultMaxRerankTokens = 8196
+
+// ErrRerankTokenLimitPolicy identifies invalid token-limit configuration or oversized rerank input.
+var ErrRerankTokenLimitPolicy = errors.New("rerank token limit policy")
+
// Message represents a chat message with role and content
//
// is interface{} to support different formats:
@@ -283,19 +293,53 @@ type RerankModel struct {
ModelDriver ModelDriver
ModelName *string
APIConfig *APIConfig
+ MaxTokens int
}
// NewRerankModel creates a new RerankModel
-func NewRerankModel(driver ModelDriver, modelName *string, apiConfig *APIConfig) *RerankModel {
+func NewRerankModel(driver ModelDriver, modelName *string, apiConfig *APIConfig, maxTokens int) *RerankModel {
+ if maxTokens <= 0 {
+ maxTokens = defaultMaxRerankTokens
+ }
return &RerankModel{
ModelDriver: driver,
ModelName: modelName,
APIConfig: apiConfig,
+ MaxTokens: maxTokens,
}
}
// Rerank calculates similarity between query and texts
func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, apiConfig *APIConfig, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error) {
+ maxTokens := r.MaxTokens
+ if maxTokens <= 0 {
+ maxTokens = defaultMaxRerankTokens
+ }
+ mode := strings.ToLower(strings.TrimSpace(common.GetEnv(common.EnvRerankTokenLimitMode)))
+ if mode == "" {
+ mode = "truncate"
+ }
+ if mode != "truncate" && mode != "passthrough" && mode != "raise_error" {
+ return nil, fmt.Errorf("%w: invalid %s %q; expected %q, %q, or %q", ErrRerankTokenLimitPolicy, common.EnvRerankTokenLimitMode, mode, "truncate", "passthrough", "raise_error")
+ }
+ if mode != "passthrough" && request.Query != "" && len(request.Documents) > 0 {
+ queryTokens := tokenizer.NumTokensFromString(request.Query)
+ if mode == "truncate" {
+ documentTokens := max(maxTokens-queryTokens, 0)
+ documents := make([]string, len(request.Documents))
+ for i, document := range request.Documents {
+ documents[i] = tokenizer.TrimContentToTokenLimit(document, documentTokens)
+ }
+ request.Documents = documents
+ } else {
+ for i, document := range request.Documents {
+ inputTokens := queryTokens + tokenizer.NumTokensFromString(document)
+ if inputTokens > maxTokens {
+ return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, maxTokens)
+ }
+ }
+ }
+ }
return r.ModelDriver.Rerank(ctx, r.ModelName, request, apiConfig, rerankConfig, modelUsage)
}
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index 8037cbbc0f..dedd4c381d 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -2024,11 +2024,11 @@ func (s *ChatPipelineService) getModels(ctx context.Context, chat *entity.Chat)
// Rerank model.
var rerankModel *modelModule.RerankModel
if chat.RerankID != "" {
- rerankDriver, rerankName, rerankConfig, _, err := s.ModelProviderSvc.ResolveModelConfig(
+ rerankDriver, rerankName, rerankConfig, maxTokens, err := s.ModelProviderSvc.ResolveModelConfig(
ctx, chat.TenantID, entity.ModelTypeRerank, chat.RerankID,
)
if err == nil {
- rerankModel = modelModule.NewRerankModel(rerankDriver, &rerankName, rerankConfig)
+ rerankModel = modelModule.NewRerankModel(rerankDriver, &rerankName, rerankConfig, maxTokens)
}
}
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 898fbf024e..9c60a5c26c 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -398,18 +398,18 @@ func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.Retrieval
// Get rerank model if RerankID is specified
var rerankModel *models.RerankModel
if req.TenantRerankID != nil && *req.TenantRerankID != "" {
- driver, mdlName, apiConfig, _, getErr := modelProviderSvc.GetModelConfigByID(ctx, tenantIDs[0], entity.ModelTypeRerank, *req.TenantRerankID)
+ driver, mdlName, apiConfig, maxTokens, getErr := modelProviderSvc.GetModelConfigByID(ctx, tenantIDs[0], entity.ModelTypeRerank, *req.TenantRerankID)
if getErr != nil {
return nil, fmt.Errorf("failed to get rerank model by tenant_rerank_id: %w", getErr)
}
- rerankModel = models.NewRerankModel(driver, &mdlName, apiConfig)
+ rerankModel = models.NewRerankModel(driver, &mdlName, apiConfig, maxTokens)
} else if req.RerankID != nil && *req.RerankID != "" {
rerankCompositeName := *req.RerankID
- driver, mdlName, apiConfig, _, getErr := modelProviderSvc.ResolveModelConfig(ctx, tenantIDs[0], entity.ModelTypeRerank, rerankCompositeName)
+ driver, mdlName, apiConfig, maxTokens, getErr := modelProviderSvc.ResolveModelConfig(ctx, tenantIDs[0], entity.ModelTypeRerank, rerankCompositeName)
if getErr != nil {
rerankModel = nil
} else {
- rerankModel = models.NewRerankModel(driver, &mdlName, apiConfig)
+ rerankModel = models.NewRerankModel(driver, &mdlName, apiConfig, maxTokens)
}
}
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go
index 3c89ff890a..486b209015 100644
--- a/internal/service/dataset/search.go
+++ b/internal/service/dataset/search.go
@@ -294,11 +294,11 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
// Get rerank model if rerankID is specified
var rerankModel *modelModule.RerankModel
if rerankID != "" {
- driver, modelName, apiConfig, _, rErr := modelProviderSvc.ResolveModelConfig(ctx, tenantIDs[0], entity.ModelTypeRerank, rerankID)
+ driver, modelName, apiConfig, maxTokens, rErr := modelProviderSvc.ResolveModelConfig(ctx, tenantIDs[0], entity.ModelTypeRerank, rerankID)
if rErr != nil {
return nil, fmt.Errorf("failed to get rerank model by rerank_id: %w", rErr)
}
- rerankModel = modelModule.NewRerankModel(driver, &modelName, apiConfig)
+ rerankModel = modelModule.NewRerankModel(driver, &modelName, apiConfig, maxTokens)
}
retrievalReq := &nlp.RetrievalRequest{
diff --git a/internal/service/model_service.go b/internal/service/model_service.go
index ea5a3c6c30..ae47ca8a6a 100644
--- a/internal/service/model_service.go
+++ b/internal/service/model_service.go
@@ -2567,6 +2567,7 @@ func modelInfoWithTenantExtra(modelInfo *modelModule.Model, modelEntity *entity.
if extra.MaxTokens != nil && *extra.MaxTokens > 0 {
model.MaxOutput = extra.MaxTokens
+ model.MaxTokens = extra.MaxTokens
}
if len(extra.ModelTypes) > 0 {
model.ModelTypes = append([]string(nil), extra.ModelTypes...)
@@ -2608,6 +2609,19 @@ func maxTokensFromTenantModelExtra(modelEntity *entity.TenantModel, fallback int
return fallback, nil
}
+func maxTokensFromModelInfo(modelInfo *modelModule.Model, modelType entity.ModelType) int {
+ if modelInfo == nil {
+ return 0
+ }
+ if (modelType == entity.ModelTypeEmbedding || modelType == entity.ModelTypeRerank) && modelInfo.MaxTokens != nil {
+ return *modelInfo.MaxTokens
+ }
+ if modelInfo.MaxOutput != nil {
+ return *modelInfo.MaxOutput
+ }
+ return 0
+}
+
func (m *ModelProviderService) getModelInstanceAndProviderByName(ctx context.Context, providerName, instanceName, modelName *string, userID string, apiConfig *modelModule.APIConfig) (*ModelInstanceAndProviderInfo, error) {
if providerName == nil || instanceName == nil || modelName == nil {
return nil, errors.New("provider name, instance name and model name are required when model id is absent")
@@ -3065,7 +3079,8 @@ func (m *ModelProviderService) RerankDocument(ctx context.Context, providerName,
}
var response *modelModule.RerankResponse
- response, err = modelDriver.Rerank(ctx, &resolvedModelName, rerankRequest, info.APIConfig, modelConfig, nil)
+ rerankModel := modelModule.NewRerankModel(modelDriver, &resolvedModelName, info.APIConfig, maxTokensFromModelInfo(info.ModelInfo, entity.ModelTypeRerank))
+ response, err = rerankModel.Rerank(ctx, rerankRequest, info.APIConfig, modelConfig, nil)
if err != nil {
return nil, common.CodeServerError, err
}
@@ -3445,11 +3460,11 @@ func (m *ModelProviderService) GetChatModel(ctx context.Context, tenantID, compo
// GetRerankModel returns a RerankModel wrapper for the given tenant
func (m *ModelProviderService) GetRerankModel(ctx context.Context, tenantID, compositeModelName string) (*modelModule.RerankModel, error) {
- driver, modelName, apiConfig, _, err := m.ResolveModelConfig(ctx, tenantID, entity.ModelTypeRerank, compositeModelName)
+ driver, modelName, apiConfig, maxTokens, err := m.ResolveModelConfig(ctx, tenantID, entity.ModelTypeRerank, compositeModelName)
if err != nil {
return nil, err
}
- return modelModule.NewRerankModel(driver, &modelName, apiConfig), nil
+ return modelModule.NewRerankModel(driver, &modelName, apiConfig, maxTokens), nil
}
type AddModelRequest struct {
@@ -3582,9 +3597,7 @@ func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID st
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerEntity.ProviderName, modelEntity.ModelName); mi != nil {
- if mi.MaxOutput != nil {
- maxTokens = *mi.MaxOutput
- }
+ maxTokens = maxTokensFromModelInfo(mi, modelType)
}
maxTokens, err = maxTokensFromTenantModelExtra(modelEntity, maxTokens)
if err != nil {
@@ -3996,11 +4009,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: &region}
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName("Builtin", pureModelName); mi != nil {
- if mi.MaxOutput == nil {
- maxTokens = 0
- } else {
- maxTokens = *mi.MaxOutput
- }
+ maxTokens = maxTokensFromModelInfo(mi, modelType)
}
return builtinDriver, pureModelName, apiConfig, maxTokens, nil
}
@@ -4058,11 +4067,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
}
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil {
- if mi.MaxOutput == nil {
- maxTokens = 0
- } else {
- maxTokens = *mi.MaxOutput
- }
+ maxTokens = maxTokensFromModelInfo(mi, modelType)
}
maxTokens, driverErr = maxTokensFromTenantModelExtra(modelObj, maxTokens)
if driverErr != nil {
@@ -4115,10 +4120,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
return nil, "", nil, 0, driverErr
}
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: &region, BaseURL: &baseURL}
- maxTokens := 0
- if llmInfo.MaxOutput != nil {
- maxTokens = *llmInfo.MaxOutput
- }
+ maxTokens := maxTokensFromModelInfo(llmInfo, modelType)
return driver, llmInfo.Name, apiConfig, maxTokens, nil
}
diff --git a/internal/service/model_service_test.go b/internal/service/model_service_test.go
index e2c601d034..a99d29123e 100644
--- a/internal/service/model_service_test.go
+++ b/internal/service/model_service_test.go
@@ -500,6 +500,21 @@ func TestModelProviderServiceGetModelConfigByID(t *testing.T) {
}
}
+func TestMaxTokensFromModelInfo(t *testing.T) {
+ maxTokens := 4096
+ maxOutput := 1024
+ modelInfo := &modelModule.Model{MaxTokens: &maxTokens, MaxOutput: &maxOutput}
+ if got := maxTokensFromModelInfo(modelInfo, entity.ModelTypeRerank); got != maxTokens {
+ t.Fatalf("rerank max tokens = %d, want %d", got, maxTokens)
+ }
+ if got := maxTokensFromModelInfo(modelInfo, entity.ModelTypeEmbedding); got != maxTokens {
+ t.Fatalf("embedding max tokens = %d, want %d", got, maxTokens)
+ }
+ if got := maxTokensFromModelInfo(modelInfo, entity.ModelTypeChat); got != maxOutput {
+ t.Fatalf("chat max tokens = %d, want max output %d", got, maxOutput)
+ }
+}
+
func TestModelProviderServiceResolveModelContextLength(t *testing.T) {
db := setupModelProviderServiceTestDB(t)
useModelProviderServiceTestDB(t, db)
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.go
index 9c2f338c97..01c3b22063 100644
--- a/internal/service/nlp/reranker.go
+++ b/internal/service/nlp/reranker.go
@@ -17,6 +17,7 @@ package nlp
import (
"context"
"encoding/json"
+ "errors"
"math"
"regexp"
"sort"
@@ -66,7 +67,7 @@ func Rerank(
cfield string,
qb *QueryBuilder,
rankFeature map[string]float64,
-) (sim []float64, tsim []float64, vsim []float64) {
+) (sim []float64, tsim []float64, vsim []float64, err error) {
// If reranker model is provided and there are results, use model reranking
if rerankModel != nil && total > 0 {
return RerankByModel(ctx, rerankModel, chunks, nil, nil, query, tkWeight, vtWeight, cfield, qb, rankFeature)
@@ -77,14 +78,16 @@ func Rerank(
// For Infinity: scores are already normalized before fusion
// Just extract the scores from results
if chunks == nil || total == 0 || len(chunks) == 0 {
- return []float64{}, []float64{}, []float64{}
+ return []float64{}, []float64{}, []float64{}, nil
}
- return RerankInfinityFallback(chunks)
+ sim, tsim, vsim = RerankInfinityFallback(chunks)
+ return sim, tsim, vsim, nil
}
// For Elasticsearch: need to perform reranking and apply rank features
- return RerankStandard(chunks, keywords, questionVector, query, tkWeight, vtWeight, cfield, qb, rankFeature)
+ sim, tsim, vsim = RerankStandard(chunks, keywords, questionVector, query, tkWeight, vtWeight, cfield, qb, rankFeature)
+ return sim, tsim, vsim, nil
}
// RerankByModel performs reranking using a reranker model
@@ -99,9 +102,9 @@ func RerankByModel(
cfield string,
qb *QueryBuilder,
rankFeature map[string]float64,
-) (sim []float64, tsim []float64, vsim []float64) {
+) (sim []float64, tsim []float64, vsim []float64, err error) {
if chunks == nil || len(chunks) == 0 {
- return []float64{}, []float64{}, []float64{}
+ return []float64{}, []float64{}, []float64{}, nil
}
chunkCount := len(chunks)
@@ -155,8 +158,11 @@ func RerankByModel(
tsim = TokenSimilarity(keywords, insTw, qb)
// Get similarity scores from reranker model
- rerankResponse, err := rerankModel.ModelDriver.Rerank(ctx, rerankModel.ModelName, models.RerankRequest{Query: query, Documents: docs}, rerankModel.APIConfig, &models.RerankConfig{}, nil)
+ rerankResponse, err := rerankModel.Rerank(ctx, models.RerankRequest{Query: query, Documents: docs}, rerankModel.APIConfig, &models.RerankConfig{}, nil)
if err != nil {
+ if errors.Is(err, models.ErrRerankTokenLimitPolicy) {
+ return nil, nil, nil, err
+ }
common.Error("RerankByModel: rerankModel.Rerank failed; falling back to token-only similarity", err)
// If model fails, fall back to token similarity only
rerankResponse = &models.RerankResponse{}
@@ -193,7 +199,7 @@ func RerankByModel(
sim = applyRankFeatureScoresForIDs(ids, field, sim, rankFeature)
common.Info("RerankByModel completed")
- return sim, tsim, modelSim
+ return sim, tsim, modelSim, nil
}
// NormalizeRerankScores rescales reranker scores into [0, 1] for the
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index 77fd476647..8b060cba42 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -156,7 +156,10 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)
return &RetrievalResult{Chunks: []map[string]interface{}{}, DocAggs: []map[string]interface{}{}, Total: 0}, nil
}
- sim, termSimilarity, vectorSimilarity := s.scoreSearchResult(ctx, req, searchResult)
+ sim, termSimilarity, vectorSimilarity, err := s.scoreSearchResult(ctx, req, searchResult)
+ if err != nil {
+ return nil, err
+ }
if len(sim) == 0 {
return &RetrievalResult{Chunks: []map[string]interface{}{}, DocAggs: []map[string]interface{}{}, Total: 0}, nil
}
@@ -408,7 +411,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)
}, nil
}
-func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *RetrievalRequest, searchResult *RetrievalSearchResult) ([]float64, []float64, []float64) {
+func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *RetrievalRequest, searchResult *RetrievalSearchResult) ([]float64, []float64, []float64, error) {
// sim = tkWeight*tsim + vtWeight*vsim
vtWeight := *req.VectorSimilarityWeight
tkWeight := 1.0 - vtWeight
@@ -445,11 +448,11 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
}
}
}
- return sim, sim, sim
+ return sim, sim, sim, nil
}
if useOceanBase {
- return RerankStandard(
+ sim, tsim, vsim := RerankStandard(
searchResult.Chunks,
nil,
searchResult.QueryVector,
@@ -460,12 +463,13 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
qb,
*req.RankFeature,
)
+ return sim, tsim, vsim, nil
}
knnResult, err := s.docEngine.KNNScores(ctx, searchResult.Chunks, searchResult.QueryVector, len(searchResult.IDs))
if err != nil {
common.Warn("KNNScores failed for ES, falling back to local computation", zap.Error(err))
- return RerankStandard(
+ sim, tsim, vsim := RerankStandard(
searchResult.Chunks,
nil,
searchResult.QueryVector,
@@ -476,9 +480,10 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
qb,
*req.RankFeature,
)
+ return sim, tsim, vsim, nil
}
knnScores := s.docEngine.GetScores(knnResult)
- return RerankWithKNN(
+ sim, tsim, vsim := RerankWithKNN(
searchResult.Chunks,
searchResult.IDs,
searchResult.Field,
@@ -490,6 +495,7 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
qb,
*req.RankFeature,
)
+ return sim, tsim, vsim, nil
}
// RetrievalSearchRequest is the request struct for RetrievalService.Search()