Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(retrieval): align pagerank and rank-feature scoring (#21542)

qinling02107c8a7f26a148
diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.go
index 1a761d7554..a911617af1 100644
--- a/internal/agent/tool/retrieval_nlp.go
+++ b/internal/agent/tool/retrieval_nlp.go
@@ -256,19 +256,7 @@ func (a *NLPRetrievalAdapter) Search(ctx context.Context, db *gorm.DB, req Retri
}
query = retrievalUserPrefixPattern.ReplaceAllString(query, "")
vectorOnly := req.KeywordsSimilarityWeight != nil && *req.KeywordsSimilarityWeight < 0.01
- // rank_feature (Python retrieve: rank_feature=label_question(question,
- // self.kbs)). Prefer a feature supplied on the request (computed by RAGTools
- // from its own KB objects) so the agentic tool stays authoritative; fall
- // back to the enhancer, which resolves the KB objects itself.
- var rankFeature map[string]float64
- if req.RankFeature != nil && len(*req.RankFeature) > 0 {
- rankFeature = *req.RankFeature
- } else if a.enhancer != nil && !vectorOnly {
- // Rank features are extra scoring clauses on the TEXT leg; a
- // vector-only request has no text leg, so labelling the question would
- // be a tag lookup whose result nothing consumes.
- rankFeature = a.enhancer.LabelQuestion(ctx, query, datasets.kbs)
- }
+ rankFeature := resolveRankFeature(ctx, req.RankFeature, a.enhancer, query, datasets.kbs, vectorOnly)
rerankModel, err := a.resolveRerankModel(ctx, req, datasets.kbs[0].TenantID)
if err != nil {
return nil, err
@@ -280,9 +268,7 @@ func (a *NLPRetrievalAdapter) Search(ctx context.Context, db *gorm.DB, req Retri
nlpReq := nlpRequestFromRetrieval(preparedReq, datasets.tenantIDs, topN, embeddingModel, preparedReq.ExcludeCompiled)
nlpReq.Language = entity.KnowledgebasesLanguage(datasets.kbs)
nlpReq.RerankModel = rerankModel
- if rankFeature != nil {
- nlpReq.RankFeature = &rankFeature
- }
+ nlpReq.RankFeature = &rankFeature
res, err := a.svc.Retrieval(ctx, nlpReq)
if err != nil {
@@ -302,6 +288,46 @@ func (a *NLPRetrievalAdapter) Search(ctx context.Context, db *gorm.DB, req Retri
return out, nil
}
+// resolveRankFeature resolves the rank features handed to the nlp layer, which
+// takes them as *map and distinguishes "unset" from "empty". The result is never
+// nil: a nil map leaves that pointer unset and the nlp layer substitutes its
+// {pagerank_fea: 10} default.
+func resolveRankFeature(
+ ctx context.Context,
+ requested *map[string]float64,
+ enhancer retrievalEnhancer,
+ query string,
+ kbs []*entity.Knowledgebase,
+ vectorOnly bool,
+) map[string]float64 {
+ // A caller-supplied feature (computed by RAGTools from its own KB objects) is
+ // authoritative: nil means "said nothing, label it", an empty map means
+ // "there are none" and must not turn into a label lookup.
+ if requested != nil {
+ return cloneRankFeature(*requested)
+ }
+ // Only the engine-side clause needs a text leg, so a dense-only request is not
+ // labelled. The features still reach the similarity - with no scored bool, the
+ // nlp layer applies them locally - so a caller-supplied feature keeps its
+ // effect in vector-only mode and only the enhancer fallback is skipped.
+ if enhancer == nil || vectorOnly {
+ return map[string]float64{}
+ }
+ if labels := enhancer.LabelQuestion(ctx, query, kbs); labels != nil {
+ return cloneRankFeature(labels)
+ }
+ return map[string]float64{}
+}
+
+// cloneRankFeature detaches the request's map from the caller's.
+func cloneRankFeature(features map[string]float64) map[string]float64 {
+ clone := make(map[string]float64, len(features))
+ for field, weight := range features {
+ clone[field] = weight
+ }
+ return clone
+}
+
func nlpRequestFromRetrieval(
req RetrievalRequest,
tenantIDs []string,
@@ -365,17 +391,6 @@ func nlpRequestFromRetrieval(
"must_not": map[string]interface{}{"exists": "compile_kwd"},
}
}
- // Restrict to ordinary document text chunks with the SAME filter grep_chunks
- // applies: available_int=1 (must_not available_int<1 semantics so chunks
- // whose available_int is absent still pass) and must_not exists compile_kwd
- // to exclude knowledge-compiled products. Setting available_int explicitly
- // here matches grep exactly (both go through buildBoolQueryFromCondition).
- if req.OnlyOriginalText {
- nlpReq.Filter = map[string]interface{}{
- "available_int": 1,
- "must_not": map[string]interface{}{"exists": "compile_kwd"},
- }
- }
return nlpReq
}
diff --git a/internal/agent/tool/retrieval_nlp_test.go b/internal/agent/tool/retrieval_nlp_test.go
index 40e14953dd..e9dc03b4ca 100644
--- a/internal/agent/tool/retrieval_nlp_test.go
+++ b/internal/agent/tool/retrieval_nlp_test.go
@@ -20,9 +20,10 @@
// nlp.RetrievalService, which requires a doc engine + document
// DAO. Those require network access and aren't worth standing up
// in a unit test. Instead we exercise the translation layer
-// (translateChunk + helpers) directly — that's the surface that
+// (translateChunk + helpers) and the rank-feature resolution
+// (resolveRankFeature) directly — that's the surface that
// differs between the nlp chunk shape and the agent-tool chunk
-// shape, and it's the only piece the adapter owns.
+// shape, and the only piece the adapter itself decides.
package tool
@@ -626,3 +627,142 @@ func (f fakeKnowledgebaseLookup) GetByName(_ context.Context, _ *gorm.DB, name,
}
return nil, gorm.ErrRecordNotFound
}
+
+// stubRankFeatureEnhancer records whether the question was labelled.
+type stubRankFeatureEnhancer struct {
+ labels map[string]float64
+ calls int
+}
+
+func (s *stubRankFeatureEnhancer) CrossLanguages(context.Context, string, string, []string) (string, error) {
+ return "", nil
+}
+
+func (s *stubRankFeatureEnhancer) FilterDocuments(
+ context.Context, map[string]any, string, *modelModule.ChatModel, []string, []string,
+) ([]string, error) {
+ return nil, nil
+}
+
+func (s *stubRankFeatureEnhancer) LabelQuestion(
+ context.Context, string, []*entity.Knowledgebase,
+) map[string]float64 {
+ s.calls++
+ return s.labels
+}
+
+func (s *stubRankFeatureEnhancer) RetrieveByChildren(
+ context.Context, []map[string]any, []string,
+) []map[string]any {
+ return nil
+}
+
+func testRankFeatureKBs() []*entity.Knowledgebase {
+ return []*entity.Knowledgebase{{ID: "kb-1", TenantID: "tenant-1"}}
+}
+
+// No tag source: labelling yields nil, which must still reach the request as an
+// empty map instead of letting the nlp layer apply its default.
+func TestResolveRankFeature_NoTagSourceReturnsEmptyMap(t *testing.T) {
+ enhancer := &stubRankFeatureEnhancer{labels: nil}
+
+ got := resolveRankFeature(context.Background(), nil, enhancer, "q", testRankFeatureKBs(), false)
+
+ if got == nil {
+ t.Fatal("rankFeature = nil, want an empty (non-nil) map")
+ }
+ if len(got) != 0 {
+ t.Fatalf("rankFeature = %v, want empty", got)
+ }
+ if enhancer.calls != 1 {
+ t.Fatalf("LabelQuestion calls = %d, want 1", enhancer.calls)
+ }
+}
+
+func TestResolveRankFeature_TagMatchIsForwarded(t *testing.T) {
+ enhancer := &stubRankFeatureEnhancer{labels: map[string]float64{"tag_1": 0.8}}
+
+ got := resolveRankFeature(context.Background(), nil, enhancer, "q", testRankFeatureKBs(), false)
+
+ if got["tag_1"] != 0.8 {
+ t.Fatalf("rankFeature = %v, want tag_1=0.8", got)
+ }
+}
+
+// A caller-supplied feature wins over the enhancer and must not trigger a label
+// lookup.
+func TestResolveRankFeature_PrefersRequestSuppliedFeature(t *testing.T) {
+ enhancer := &stubRankFeatureEnhancer{labels: map[string]float64{"from_enhancer": 0.1}}
+ requested := map[string]float64{"from_request": 0.5}
+
+ got := resolveRankFeature(context.Background(), &requested, enhancer, "q", testRankFeatureKBs(), false)
+
+ if got["from_request"] != 0.5 || len(got) != 1 {
+ t.Fatalf("rankFeature = %v, want only from_request=0.5", got)
+ }
+ if enhancer.calls != 0 {
+ t.Fatalf("LabelQuestion calls = %d, want 0", enhancer.calls)
+ }
+}
+
+// A vector-only request skips the label lookup; the map must still be non-nil.
+func TestResolveRankFeature_VectorOnlySkipsLabelling(t *testing.T) {
+ enhancer := &stubRankFeatureEnhancer{labels: map[string]float64{"tag_1": 0.8}}
+
+ got := resolveRankFeature(context.Background(), nil, enhancer, "q", testRankFeatureKBs(), true)
+
+ if got == nil {
+ t.Fatal("rankFeature = nil, want an empty (non-nil) map")
+ }
+ if len(got) != 0 {
+ t.Fatalf("rankFeature = %v, want empty", got)
+ }
+ if enhancer.calls != 0 {
+ t.Fatalf("LabelQuestion calls = %d, want 0", enhancer.calls)
+ }
+}
+
+func TestResolveRankFeature_NilEnhancerReturnsEmptyMap(t *testing.T) {
+ got := resolveRankFeature(context.Background(), nil, nil, "q", testRankFeatureKBs(), false)
+
+ if got == nil {
+ t.Fatal("rankFeature = nil, want an empty (non-nil) map")
+ }
+ if len(got) != 0 {
+ t.Fatalf("rankFeature = %v, want empty", got)
+ }
+}
+
+// nil and {} are distinct: nil asks for labelling, {} means there are none.
+func TestResolveRankFeature_ExplicitEmptyMapSuppressesLabelling(t *testing.T) {
+ enhancer := &stubRankFeatureEnhancer{labels: map[string]float64{"tag_1": 0.8}}
+ requested := map[string]float64{}
+
+ got := resolveRankFeature(context.Background(), &requested, enhancer, "q", testRankFeatureKBs(), false)
+
+ if got == nil {
+ t.Fatal("rankFeature = nil, want an empty (non-nil) map")
+ }
+ if len(got) != 0 {
+ t.Fatalf("rankFeature = %v, want empty", got)
+ }
+ if enhancer.calls != 0 {
+ t.Fatalf("LabelQuestion calls = %d, want 0", enhancer.calls)
+ }
+}
+
+// The result must not alias the caller's map.
+func TestResolveRankFeature_ReturnsDetachedCopy(t *testing.T) {
+ requested := map[string]float64{"tag_1": 0.4}
+
+ got := resolveRankFeature(context.Background(), &requested, nil, "q", testRankFeatureKBs(), false)
+ got["tag_1"] = 0.9
+ got["injected"] = 1.0
+
+ if requested["tag_1"] != 0.4 {
+ t.Fatalf("caller map mutated through the result: %v", requested)
+ }
+ if _, ok := requested["injected"]; ok {
+ t.Fatalf("caller map gained a key through the result: %v", requested)
+ }
+}
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index 6034e9d49c..f27dfef5ae 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -1318,7 +1318,7 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
"k": k,
"num_candidates": numCandidates,
"similarity": similarity,
- "filter": boolQuery,
+ "filter": cloneFilterSnapshot(boolQuery),
}
queryBody["knn"] = knnQuery
@@ -1345,17 +1345,13 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
if rankFeatureQuery != nil {
if boolQuery, ok := queryBody["query"].(map[string]interface{}); ok {
if boolMap, ok := boolQuery["bool"].(map[string]interface{}); ok {
- if should, ok := boolMap["should"].([]interface{}); ok {
- for _, q := range rankFeatureQuery {
- boolMap["should"] = append(should, q)
- }
- } else {
- interfaceSlice := make([]interface{}, len(rankFeatureQuery))
- for i, q := range rankFeatureQuery {
- interfaceSlice[i] = q
- }
- boolMap["should"] = interfaceSlice
+ // Accumulate into a local slice first: re-appending to the
+ // pre-loop header keeps only the last rank_feature clause.
+ should, _ := boolMap["should"].([]interface{})
+ for _, q := range rankFeatureQuery {
+ should = append(should, q)
}
+ boolMap["should"] = should
}
}
}
@@ -1497,15 +1493,10 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
// back empty here, which flattened every hybrid chunk's score to 0.
scoreColumn := "_score"
- pagerankField := common.PAGERANK_FLD
- if isSkillIndex {
- pagerankField = ""
- }
- if isMemoryIndex {
- pagerankField = ""
- }
-
- allResults = calculateScores(allResults, scoreColumn, pagerankField)
+ // pagerank reaches the ES hit score only through the rank_feature query
+ // built above, and reaches the final similarity in the reranker. Adding
+ // it to _score here as well would count it twice.
+ allResults = calculateScores(allResults, scoreColumn, "")
allResults = sortByScore(allResults, limit)
}
@@ -2145,6 +2136,35 @@ func memoryMessageStatusBool(value interface{}) bool {
}
}
+// cloneFilterSnapshot freezes the bool query's shape at the point the kNN clause
+// is built. The reference does the same by passing bool_query.to_dict() as the
+// kNN filter; the rank_feature clauses are appended to the same bool afterwards
+// and belong to the scored query only.
+func cloneFilterSnapshot(q map[string]interface{}) map[string]interface{} {
+ if q == nil {
+ return nil
+ }
+ clone := make(map[string]interface{}, len(q))
+ for k, v := range q {
+ clone[k] = v
+ }
+ inner, ok := q["bool"].(map[string]interface{})
+ if !ok {
+ return clone
+ }
+ innerClone := make(map[string]interface{}, len(inner))
+ for k, v := range inner {
+ innerClone[k] = v
+ }
+ for _, key := range []string{"must", "should", "filter", "must_not"} {
+ if clauses, ok := inner[key].([]interface{}); ok {
+ innerClone[key] = slices.Clone(clauses)
+ }
+ }
+ clone["bool"] = innerClone
+ return clone
+}
+
// buildBoolQueryFromCondition builds an ES bool query from condition map.
// Skill indexes use status, regular chunk indexes use kb_id, and memory
// message indexes use memory_id plus message-specific storage fields.
@@ -2156,7 +2176,6 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strin
var mustClauses []interface{}
var mustNotClauses []interface{}
var filterClauses []interface{}
- var shouldClauses []interface{}
// Memory message indexes use memory_id, regular chunk indexes use kb_id.
if kbIDs != nil && len(kbIDs) > 0 {
@@ -2279,36 +2298,26 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strin
continue
}
if k == "id" {
- if v == nil || v == "" {
- continue
- }
- if listVal, ok := v.([]interface{}); ok && len(listVal) > 0 {
- shouldClauses = append(shouldClauses,
- map[string]interface{}{"terms": map[string]interface{}{"id": listVal}},
- map[string]interface{}{"terms": map[string]interface{}{"_id": listVal}},
- )
- } else if strListVal, ok := v.([]string); ok && len(strListVal) > 0 {
- // A typed []string MUST be handled here: the generic loop
- // below skips the "id" key, so without this branch a caller
- // passing map[string]interface{}{"id": []string{...}} builds a
- // query with NO id filter and the scoped read silently
- // degenerates into "fetch up to Limit chunks of the document"
- // (observed via list_chunks: an 11-chunk window returned 3.8MB
- // because the whole 4171-chunk document was fetched instead).
- shouldClauses = append(shouldClauses,
- map[string]interface{}{"terms": map[string]interface{}{"id": strListVal}},
- map[string]interface{}{"terms": map[string]interface{}{"_id": strListVal}},
- )
- } else if strVal, ok := v.(string); ok && strVal != "" {
- shouldClauses = append(shouldClauses,
- map[string]interface{}{"term": map[string]interface{}{"id": strVal}},
- map[string]interface{}{"term": map[string]interface{}{"_id": strVal}},
- )
- } else if intVal, ok := v.(int); ok && intVal != 0 {
- shouldClauses = append(shouldClauses,
- map[string]interface{}{"term": map[string]interface{}{"id": intVal}},
- map[string]interface{}{"term": map[string]interface{}{"_id": intVal}},
- )
+ // id is a scope restriction, never a scoring hint. It has to be a
+ // mandatory clause: encoding it in bool.should relied on
+ // minimum_should_match to make it binding, and Search() appends
+ // rank_feature clauses to that same top-level should list — every
+ // document carrying the rank feature then satisfied the id scope,
+ // so a chunk-id-scoped read silently returned unrelated chunks.
+ //
+ // The generic loop below skips the "id" key, so a value shape this
+ // helper does not understand leaves the query without any id
+ // filter: list_chunks then degenerates into "fetch up to Limit
+ // chunks of the document" (an 11-chunk window returned 3.8MB
+ // instead of ~17KB). idTermClauses therefore accepts every list
+ // flavour, not only []interface{}.
+ if idClauses := idTermClauses(v); len(idClauses) > 0 {
+ filterClauses = append(filterClauses, map[string]interface{}{
+ "bool": map[string]interface{}{
+ "should": idClauses,
+ "minimum_should_match": 1,
+ },
+ })
}
continue
}
@@ -2361,10 +2370,6 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strin
if len(filterClauses) > 0 {
boolQuery["filter"] = filterClauses
}
- if len(shouldClauses) > 0 {
- boolQuery["should"] = shouldClauses
- boolQuery["minimum_should_match"] = 1
- }
if len(boolQuery) == 0 {
return nil
@@ -2373,6 +2378,43 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strin
return map[string]interface{}{"bool": boolQuery}
}
+// idTermClauses renders one id condition as id/_id alternatives, so a caller
+// matches whichever field it addressed: the stored id field or the
+// Elasticsearch _id. The caller makes the result mandatory.
+//
+// The query shapes mirror the Python reference (rag/utils/es_conn.py): a
+// scalar id is a term, a list of ids is a terms query.
+func idTermClauses(value interface{}) []interface{} {
+ switch val := value.(type) {
+ case nil:
+ return nil
+ case string:
+ if val == "" {
+ return nil
+ }
+ return []interface{}{
+ map[string]interface{}{"term": map[string]interface{}{"id": val}},
+ map[string]interface{}{"term": map[string]interface{}{"_id": val}},
+ }
+ case int:
+ if val == 0 {
+ return nil
+ }
+ return []interface{}{
+ map[string]interface{}{"term": map[string]interface{}{"id": val}},
+ map[string]interface{}{"term": map[string]interface{}{"_id": val}},
+ }
+ }
+ items, ok := conditionTermsList(value)
+ if !ok || len(items) == 0 {
+ return nil
+ }
+ return []interface{}{
+ map[string]interface{}{"terms": map[string]interface{}{"id": items}},
+ map[string]interface{}{"terms": map[string]interface{}{"_id": items}},
+ }
+}
+
// buildQueryStringQuery builds a query_string query from MatchTextExpr
// When isSkillIndex is true, uses skill-specific fields (name_tks, tags_tks, etc.)
// Otherwise uses document fields (title_tks, content_ltks, etc.)
@@ -2470,14 +2512,16 @@ func buildRankFeatureQuery(rankFeature map[string]float64) []map[string]interfac
var queries []map[string]interface{}
for _, fld := range keys {
+ sc := rankFeature[fld]
+ field := fmt.Sprintf("%s.%s", common.TAG_FLD, fld)
if fld == common.PAGERANK_FLD {
- continue
+ // pagerank_fea is a top-level rank_feature field (the "*_fea" dynamic
+ // template), not one key inside the tag_feas map.
+ field = common.PAGERANK_FLD
}
- sc := rankFeature[fld]
- tagField := fmt.Sprintf("%s.%s", common.TAG_FLD, fld)
queries = append(queries, map[string]interface{}{
"rank_feature": map[string]interface{}{
- "field": tagField,
+ "field": field,
"linear": map[string]interface{}{},
"boost": sc,
},
@@ -2642,6 +2686,17 @@ func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) map
continue
}
+ if mapVal, ok := val.(map[string]interface{}); ok {
+ // Nested objects (tag_feas is a "rank_features" object) cannot
+ // take the string conversion below: "%v" renders map[k:v], which
+ // no rank-feature parser accepts. JSON is the string form the
+ // reranker parses.
+ if raw, err := json.Marshal(mapVal); err == nil {
+ m[field] = string(raw)
+ continue
+ }
+ }
+
if listVal, ok := val.([]interface{}); ok {
if len(listVal) == 1 {
if _, isArray := listVal[0].([]interface{}); !isArray {
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go
index 4f2b38935a..ae5b3f89a2 100644
--- a/internal/engine/elasticsearch/chunk_helpers_test.go
+++ b/internal/engine/elasticsearch/chunk_helpers_test.go
@@ -545,6 +545,62 @@ func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) {
}
}
+// TestBuildRankFeatureQueryUsesTopLevelPagerank pins the field naming: tag
+// weights address keys inside the tag_feas map, while pagerank addresses the
+// top-level pagerank_fea rank_feature field.
+func TestBuildRankFeatureQueryUsesTopLevelPagerank(t *testing.T) {
+ queries := buildRankFeatureQuery(map[string]float64{
+ common.PAGERANK_FLD: 10,
+ "tag_a": 0.8,
+ })
+ if len(queries) != 2 {
+ t.Fatalf("queries = %d, want 2", len(queries))
+ }
+ byField := make(map[string]map[string]interface{}, len(queries))
+ for _, q := range queries {
+ clause, ok := q["rank_feature"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("not a rank_feature clause: %#v", q)
+ }
+ field, _ := clause["field"].(string)
+ byField[field] = clause
+ }
+ pagerank, ok := byField[common.PAGERANK_FLD]
+ if !ok {
+ t.Fatalf("pagerank clause missing, got fields %#v", byField)
+ }
+ if got := pagerank["boost"]; got != float64(10) {
+ t.Fatalf("pagerank boost = %v, want 10", got)
+ }
+ tagField := common.TAG_FLD + ".tag_a"
+ if _, ok := byField[tagField]; !ok {
+ t.Fatalf("tag clause %q missing, got fields %#v", tagField, byField)
+ }
+}
+
+// TestElasticsearchGetFieldsEncodesNestedObjects covers the rank-feature
+// fields: tag_feas is a "rank_features" object in _source, so the projection
+// has to render it as JSON (the reranker parses JSON, not Go's "%v" map form),
+// while numeric fields keep the stringified form the rest of the projection uses.
+func TestElasticsearchGetFieldsEncodesNestedObjects(t *testing.T) {
+ engine := &Engine{}
+ chunks := []map[string]interface{}{
+ {
+ "id": "chunk-1",
+ "tag_feas": map[string]interface{}{"tag_a": float64(0.5)},
+ "pagerank_fea": float64(7),
+ "content_with_weight": "body",
+ },
+ }
+
+ fieldMap, ok := engine.GetFields(chunks, []string{"id", "tag_feas", "pagerank_fea"})["chunk-1"]
+ if !ok {
+ t.Fatalf("GetFields lost chunk-1")
+ }
+ assertEqual(t, fieldMap["tag_feas"], `{"tag_a":0.5}`)
+ assertEqual(t, fieldMap["pagerank_fea"], "7")
+}
+
func TestElasticsearchGetFieldsEmptyAndSkippedIDs(t *testing.T) {
engine := &Engine{}
@@ -760,6 +816,187 @@ func TestMapMemoryMessageToESDoc(t *testing.T) {
}
}
+// TestSearchKeepsRankFeatureOutOfKNNFilter pins the body-level split: the
+// rank_feature clauses belong to the scored query, while knn.filter is a snapshot
+// of the bool taken when the kNN clause is built and must keep only the scope and
+// lexical predicates.
+func TestSearchKeepsRankFeatureOutOfKNNFilter(t *testing.T) {
+ if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {
+ t.Fatalf("init logger: %v", err)
+ }
+ var searchQuery map[string]interface{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if err := json.NewDecoder(r.Body).Decode(&searchQuery); err != nil {
+ t.Errorf("decode search query: %v", err)
+ }
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"hits":{"total":{"value":0},"hits":[]}}`))
+ }))
+ defer server.Close()
+
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ _, err = engine.Search(t.Context(), &types.SearchRequest{
+ IndexNames: []string{"ragflow_tenant"},
+ KbIDs: []string{"kb-1"},
+ Limit: 10,
+ Filter: map[string]interface{}{"available_int": 1},
+ MatchExprs: []interface{}{
+ &types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},
+ &types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},
+ &types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},
+ },
+ RankFeature: map[string]float64{"pagerank_fea": 10},
+ })
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+
+ queryJSON, err := json.Marshal(searchQuery["query"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(queryJSON), "rank_feature") || !strings.Contains(string(queryJSON), "pagerank_fea") {
+ t.Fatalf("scored query lost its rank_feature clause: %s", queryJSON)
+ }
+
+ knn, ok := searchQuery["knn"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("KNN query missing from %#v", searchQuery)
+ }
+ filterJSON, err := json.Marshal(knn["filter"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(string(filterJSON), "rank_feature") {
+ t.Fatalf("knn.filter carries rank_feature clauses: %s", filterJSON)
+ }
+ // The snapshot must still be the scope + lexical predicate, not an empty one.
+ if !strings.Contains(string(filterJSON), "query_string") {
+ t.Fatalf("knn.filter lost the lexical predicate: %s", filterJSON)
+ }
+}
+
+// TestSearchKeepsEveryRankFeatureClause covers the multi-feature case: appending
+// to the pre-loop should header used to keep only the last clause.
+func TestSearchKeepsEveryRankFeatureClause(t *testing.T) {
+ if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {
+ t.Fatalf("init logger: %v", err)
+ }
+ var searchQuery map[string]interface{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if err := json.NewDecoder(r.Body).Decode(&searchQuery); err != nil {
+ t.Errorf("decode search query: %v", err)
+ }
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"hits":{"total":{"value":0},"hits":[]}}`))
+ }))
+ defer server.Close()
+
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ _, err = engine.Search(t.Context(), &types.SearchRequest{
+ IndexNames: []string{"ragflow_tenant"},
+ KbIDs: []string{"kb-1"},
+ Limit: 10,
+ Filter: map[string]interface{}{"available_int": 1},
+ MatchExprs: []interface{}{
+ &types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},
+ &types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},
+ &types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},
+ },
+ RankFeature: map[string]float64{"tag_1": 0.8, "tag_2": 0.5},
+ })
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+
+ query, ok := searchQuery["query"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("query missing from %#v", searchQuery)
+ }
+ boolMap, ok := query["bool"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("bool query missing from %#v", query)
+ }
+ should, _ := boolMap["should"].([]interface{})
+ var fields []string
+ for _, clause := range should {
+ clauseMap, ok := clause.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ rankFeature, ok := clauseMap["rank_feature"].(map[string]interface{})
+ if !ok {
+ continue
+ }
+ if field, ok := rankFeature["field"].(string); ok {
+ fields = append(fields, field)
+ }
+ }
+ if len(fields) != 2 {
+ t.Fatalf("rank_feature clauses = %v, want both tag fields", fields)
+ }
+ joined := strings.Join(fields, ",")
+ if !strings.Contains(joined, "tag_feas.tag_1") || !strings.Contains(joined, "tag_feas.tag_2") {
+ t.Fatalf("rank_feature fields = %v, want tag_feas.tag_1 and tag_feas.tag_2", fields)
+ }
+}
+
+// TestSearchDoesNotAddPageRankToHitScore pins the hybrid score contract: _score is
+// the ES hit score and nothing else. pagerank already reaches the hit score
+// through the rank_feature clause and the final similarity through the reranker;
+// folding it in here again would count it twice.
+func TestSearchDoesNotAddPageRankToHitScore(t *testing.T) {
+ if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {
+ t.Fatalf("init logger: %v", err)
+ }
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"hits":{"total":{"value":1},"hits":[{"_id":"chunk-1","_index":"ragflow_tenant","_score":0.87,"_source":{"content_with_weight":"alpha","pagerank_fea":69,"tag_feas":{"tag_1":0.5}}}]}}`))
+ }))
+ defer server.Close()
+
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ res, err := engine.Search(t.Context(), &types.SearchRequest{
+ IndexNames: []string{"ragflow_tenant"},
+ KbIDs: []string{"kb-1"},
+ Limit: 10,
+ Filter: map[string]interface{}{"available_int": 1},
+ MatchExprs: []interface{}{
+ &types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},
+ &types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},
+ &types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},
+ },
+ RankFeature: map[string]float64{"pagerank_fea": 10},
+ })
+ if err != nil {
+ t.Fatalf("Search: %v", err)
+ }
+ if len(res.Chunks) != 1 {
+ t.Fatalf("Search returned %d chunks, want 1", len(res.Chunks))
+ }
+ if got, ok := res.Chunks[0]["_score"].(float64); !ok || got != 0.87 {
+ t.Fatalf("_score = %#v, want the hit score 0.87 (pagerank must not be folded in)", res.Chunks[0]["_score"])
+ }
+ if got, ok := res.Chunks[0]["pagerank_fea"].(float64); !ok || got != 69 {
+ t.Fatalf("pagerank_fea = %#v, want 69 preserved for the reranker", res.Chunks[0]["pagerank_fea"])
+ }
+}
+
// TestMapMemoryMessageESUpdateFieldsRefreshesTokens: writing content via
// the update path recomputes tokenized_content_ltks before write, same
// as the Python update path.
@@ -776,3 +1013,50 @@ func TestMapMemoryMessageESUpdateFieldsRefreshesTokens(t *testing.T) {
}
assertEqual(t, doc["status_int"], 0)
}
+
+func TestCloneFilterSnapshotIsolatesLaterShouldAppends(t *testing.T) {
+ boolMap := map[string]interface{}{
+ "must": []interface{}{map[string]interface{}{"term": map[string]interface{}{"kb_id": "kb-1"}}},
+ "should": []interface{}{map[string]interface{}{"match_all": map[string]interface{}{}}},
+ }
+ boolQuery := map[string]interface{}{"bool": boolMap}
+
+ snapshot := cloneFilterSnapshot(boolQuery)
+
+ // Mirror what the rank_feature block does after the kNN clause is built: it
+ // appends to the scored query's bool/should.
+ boolMap["should"] = append(
+ boolMap["should"].([]interface{}),
+ map[string]interface{}{"rank_feature": map[string]interface{}{"field": "pagerank_fea", "boost": 10.0}},
+ )
+
+ if got := len(boolMap["should"].([]interface{})); got != 2 {
+ t.Fatalf("original should has %d clauses, want 2", got)
+ }
+ snapshotBool, ok := snapshot["bool"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("snapshot bool missing from %#v", snapshot)
+ }
+ if got := len(snapshotBool["should"].([]interface{})); got != 1 {
+ t.Fatalf("snapshot should has %d clauses, want 1 (rank_feature must stay out of knn.filter)", got)
+ }
+ if got := len(snapshotBool["must"].([]interface{})); got != 1 {
+ t.Fatalf("snapshot must has %d clauses, want 1", got)
+ }
+}
+
+func TestCloneFilterSnapshotNil(t *testing.T) {
+ if got := cloneFilterSnapshot(nil); got != nil {
+ t.Fatalf("cloneFilterSnapshot(nil) = %#v, want nil", got)
+ }
+}
+
+func TestCloneFilterSnapshotNonBoolQuery(t *testing.T) {
+ query := map[string]interface{}{
+ "query_string": map[string]interface{}{"query": "alpha"},
+ }
+ snapshot := cloneFilterSnapshot(query)
+ if snapshot["query_string"] == nil {
+ t.Fatalf("snapshot = %#v, want the query_string clause preserved", snapshot)
+ }
+}
diff --git a/internal/engine/elasticsearch/chunk_test.go b/internal/engine/elasticsearch/chunk_test.go
index a4c5ca2837..e5f0c627a0 100644
--- a/internal/engine/elasticsearch/chunk_test.go
+++ b/internal/engine/elasticsearch/chunk_test.go
@@ -20,9 +20,12 @@ import (
"bytes"
"context"
"encoding/json"
+ "net/http"
+ "net/http/httptest"
"strings"
"testing"
+ "github.com/elastic/go-elasticsearch/v8"
"ragflow/internal/common"
"ragflow/internal/engine/types"
)
@@ -397,17 +400,45 @@ func TestBuildBoolQueryFromConditionUsesLegacyGraphFallback(t *testing.T) {
// `should: [{...}, {...}]`. ES rejects the nested form as malformed.
// This test pins down the flat shape for the list, string, and int
// branches.
+//
+// The id restriction must also stay out of the top-level bool.should:
+// Search appends rank_feature clauses to that slot, and a top-level
+// minimum_should_match=1 then let any rank-feature-carrying document satisfy
+// the id scope. id is emitted as a mandatory filter clause whose nested bool
+// keeps the id/_id alternative, exactly as the Python reference does
+// (rag/utils/es_conn.py: scalar id -> term, list -> terms).
func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {
- check := func(name string, cond map[string]interface{}, wantFields []string) {
+ check := func(name string, cond map[string]interface{}, wantFields []string, wantOp string) {
t.Helper()
got := buildBoolQueryFromCondition(cond, nil, false, false)
outer, ok := got["bool"].(map[string]interface{})
if !ok {
t.Fatalf("%s: missing bool wrapper: %v", name, got)
}
- should, ok := outer["should"].([]interface{})
+ if _, exists := outer["should"]; exists {
+ t.Fatalf("%s: id leaked into the top-level should slot: %v", name, outer["should"])
+ }
+ if _, exists := outer["minimum_should_match"]; exists {
+ t.Fatalf("%s: top-level minimum_should_match would also bind rank_feature clauses", name)
+ }
+ filter, ok := outer["filter"].([]interface{})
+ if !ok || len(filter) != 1 {
+ t.Fatalf("%s: filter = %v, want the single id restriction", name, outer["filter"])
+ }
+ wrapper, ok := filter[0].(map[string]interface{})
if !ok {
- t.Fatalf("%s: should is not []interface{}: %T", name, outer["should"])
+ t.Fatalf("%s: filter[0] is %T, want map[string]interface{}", name, filter[0])
+ }
+ inner, ok := wrapper["bool"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("%s: filter[0] carries no bool: %v", name, wrapper)
+ }
+ if inner["minimum_should_match"] != 1 {
+ t.Fatalf("%s: id bool minimum_should_match = %v, want 1", name, inner["minimum_should_match"])
+ }
+ should, ok := inner["should"].([]interface{})
+ if !ok {
+ t.Fatalf("%s: should is not []interface{}: %T", name, inner["should"])
}
if len(should) != len(wantFields) {
t.Fatalf("%s: should length = %d, want %d (raw=%v)", name, len(should), len(wantFields), should)
@@ -420,19 +451,13 @@ func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {
if !ok {
t.Fatalf("%s: should[%d] is %T, want map[string]interface{} (nested array bug) — raw=%v", name, i, el, el)
}
- if inner, ok := m["term"].(map[string]interface{}); ok {
- for f := range inner {
- seenFields[f] = true
- }
- continue
+ clause, ok := m[wantOp].(map[string]interface{})
+ if !ok {
+ t.Fatalf("%s: should[%d] = %v, want %q clause", name, i, m, wantOp)
}
- if terms, ok := m["terms"].(map[string]interface{}); ok {
- for f := range terms {
- seenFields[f] = true
- }
- continue
+ for f := range clause {
+ seenFields[f] = true
}
- t.Fatalf("%s: should[%d] missing term/terms: %v", name, i, m)
}
for _, want := range wantFields {
if !seenFields[want] {
@@ -443,19 +468,19 @@ func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {
check("list_value", map[string]interface{}{
"id": []interface{}{"a", "b", "c"},
- }, []string{"id", "_id"})
+ }, []string{"id", "_id"}, "terms")
check("string_list_value", map[string]interface{}{
"id": []string{"a", "b", "c"},
- }, []string{"id", "_id"})
+ }, []string{"id", "_id"}, "terms")
check("string_value", map[string]interface{}{
"id": "doc-42",
- }, []string{"id", "_id"})
+ }, []string{"id", "_id"}, "term")
check("int_value", map[string]interface{}{
"id": 42,
- }, []string{"id", "_id"})
+ }, []string{"id", "_id"}, "term")
// A typed []string must be handled too: callers built from typed helpers
// (e.g. list_chunks' ChunkScope) pass []string, and the generic loop below
@@ -464,7 +489,69 @@ func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {
// window returned 3.8MB instead of ~17KB).
check("string_slice_value", map[string]interface{}{
"id": []string{"a", "b", "c"},
- }, []string{"id", "_id"})
+ }, []string{"id", "_id"}, "terms")
+}
+
+// TestRankFeatureDoesNotWidenChunkIDScope pins why id must be a filter clause:
+// Search appends rank_feature queries to bool.should, so an id restriction
+// living in that slot is satisfied by the rank feature alone once
+// minimum_should_match=1 is set, and a chunk-id-scoped read returns unrelated
+// chunks.
+func TestRankFeatureDoesNotWidenChunkIDScope(t *testing.T) {
+ var captured map[string]interface{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if err := json.NewDecoder(r.Body).Decode(&captured); err != nil {
+ t.Error(err)
+ http.Error(w, "bad query", http.StatusBadRequest)
+ return
+ }
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(map[string]interface{}{
+ "hits": map[string]interface{}{
+ "total": map[string]interface{}{"value": 0},
+ "hits": []interface{}{},
+ },
+ }); err != nil {
+ t.Error(err)
+ }
+ }))
+ defer server.Close()
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ eng := &Engine{client: client}
+ if _, err := eng.Search(context.Background(), &types.SearchRequest{
+ IndexNames: []string{"ragflow_t1"},
+ KbIDs: []string{"kb1"},
+ Limit: 10,
+ Filter: map[string]interface{}{"id": []string{"chunk-a"}},
+ RankFeature: map[string]float64{common.PAGERANK_FLD: 1},
+ }); err != nil {
+ t.Fatal(err)
+ }
+ query, ok := captured["query"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("captured body carries no query: %v", captured)
+ }
+ boolMap, ok := query["bool"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("query carries no bool: %v", query)
+ }
+ if _, exists := boolMap["minimum_should_match"]; exists {
+ t.Fatalf("top-level minimum_should_match also binds the rank_feature clauses: %v", boolMap)
+ }
+ encoded, err := json.Marshal(boolMap["filter"])
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(string(encoded), `"id":["chunk-a"]`) {
+ t.Fatalf("id restriction is not a mandatory filter clause: %s", encoded)
+ }
+ if !strings.Contains(string(encoded), `"minimum_should_match":1`) {
+ t.Fatalf("id filter lost the id/_id alternative: %s", encoded)
+ }
}
// paginationGRID mirrors the (page_size, top) grid from
diff --git a/internal/handler/dify_retrieval_handler.go b/internal/handler/dify_retrieval_handler.go
index 9ee74eb6b3..4e1c8383e6 100644
--- a/internal/handler/dify_retrieval_handler.go
+++ b/internal/handler/dify_retrieval_handler.go
@@ -262,7 +262,9 @@ func (h *DifyRetrievalHandler) Retrieval(c *gin.Context) {
docIDs = []string{service.NoMatchDocIDSentinel}
}
- // Label question for rank features
+ // Hand the label result over unconditionally: an empty map says "issue no
+ // rank-feature clause", while a nil pointer would let the nlp layer substitute
+ // its {pagerank_fea: 10} default and add a clause the reference never sends.
kbs := []*entity.Knowledgebase{kb}
rankFeature := h.metadataSvc.LabelQuestion(ctx, req.Query, kbs)
@@ -283,9 +285,7 @@ func (h *DifyRetrievalHandler) Retrieval(c *gin.Context) {
EmbeddingModel: embModel,
Language: entity.KnowledgebasesLanguage(kbs),
}
- if rankFeature != nil {
- sr.RankFeature = &rankFeature
- }
+ sr.RankFeature = &rankFeature
result, err := h.retrievalSvc.Retrieval(c.Request.Context(), sr)
if err != nil {
diff --git a/internal/handler/dify_retrieval_handler_test.go b/internal/handler/dify_retrieval_handler_test.go
index e9259729ab..c00b5cc936 100644
--- a/internal/handler/dify_retrieval_handler_test.go
+++ b/internal/handler/dify_retrieval_handler_test.go
@@ -557,3 +557,78 @@ func TestDifyRetrieval_TopKOutOfRange(t *testing.T) {
})
}
}
+
+// captureRetrievalRequest installs a retrieval service that records the request
+// it was handed, so a test can assert on the nlp-layer parameters.
+func captureRetrievalRequest(h *DifyRetrievalHandler) **nlp.RetrievalRequest {
+ captured := new(*nlp.RetrievalRequest)
+ h.retrievalSvc = &mockRetrievalService{
+ retrievalFn: func(ctx context.Context, req *nlp.RetrievalRequest) (*nlp.RetrievalResult, error) {
+ *captured = req
+ return &nlp.RetrievalResult{
+ Chunks: []map[string]interface{}{
+ {"doc_id": "doc1", "docnm_kwd": "Test Doc", "content_with_weight": "test content", "similarity": 0.85},
+ },
+ }, nil
+ },
+ }
+ return captured
+}
+
+// LabelQuestion returning nil (no tag source on the dataset) must still reach the
+// retrieval layer as a non-nil pointer to an empty map; a nil pointer makes the
+// nlp layer substitute its {pagerank_fea: 10} default.
+func TestDifyRetrieval_NoTagSourceSendsEmptyRankFeature(t *testing.T) {
+ h, r := setupDifyTest("user1")
+ h.metadataSvc = &mockMetadataService{
+ labelQuestionFn: func(ctx context.Context, question string, kbs []*entity.Knowledgebase) map[string]float64 {
+ return nil
+ },
+ }
+ captured := captureRetrievalRequest(h)
+
+ body := `{"knowledge_id": "kb1", "query": "test question"}`
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("POST", "/api/v1/dify/retrieval", strings.NewReader(body))
+ req.Header.Set("Content-Type", "application/json")
+ r.ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+ if *captured == nil {
+ t.Fatal("retrieval service was not called")
+ }
+ if (*captured).RankFeature == nil {
+ t.Fatal("RankFeature = nil, want a pointer to an empty map; nil lets the nlp layer substitute the pagerank_fea default")
+ }
+ if got := len(*(*captured).RankFeature); got != 0 {
+ t.Fatalf("RankFeature has %d entries (%v), want empty", got, *(*captured).RankFeature)
+ }
+}
+
+func TestDifyRetrieval_TagSourceForwardsRankFeatures(t *testing.T) {
+ h, r := setupDifyTest("user1")
+ h.metadataSvc = &mockMetadataService{
+ labelQuestionFn: func(ctx context.Context, question string, kbs []*entity.Knowledgebase) map[string]float64 {
+ return map[string]float64{"tag_1": 0.8}
+ },
+ }
+ captured := captureRetrievalRequest(h)
+
+ body := `{"knowledge_id": "kb1", "query": "test question"}`
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("POST", "/api/v1/dify/retrieval", strings.NewReader(body))
+ req.Header.Set("Content-Type", "application/json")
+ r.ServeHTTP(w, req)
+
+ if w.Code != http.StatusOK {
+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())
+ }
+ if *captured == nil || (*captured).RankFeature == nil {
+ t.Fatal("RankFeature was not forwarded to the retrieval layer")
+ }
+ if got := (*(*captured).RankFeature)["tag_1"]; got != 0.8 {
+ t.Fatalf("RankFeature[tag_1] = %v, want 0.8", got)
+ }
+}
diff --git a/internal/harness/core/cancel_full_test.go b/internal/harness/core/cancel_full_test.go
index 2d89fd8d40..582c9f49da 100644
--- a/internal/harness/core/cancel_full_test.go
+++ b/internal/harness/core/cancel_full_test.go
@@ -1238,6 +1238,21 @@ func TestAgentCancelFunc_MultiCall_JoinSafePointModes(t *testing.T) {
}
}
+// waitUntil polls cond until it holds or limit elapses. The timeout escalation
+// runs on its own goroutine, so a fixed sleep races with scheduler starvation
+// when the whole test suite runs in parallel.
+func waitUntil(t *testing.T, limit time.Duration, cond func() bool) bool {
+ t.Helper()
+ deadline := time.Now().Add(limit)
+ for time.Now().Before(deadline) {
+ if cond() {
+ return true
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ return cond()
+}
+
func TestAgentCancelFunc_MultiCall_TimeoutEscalationReturnsErrCancelTimeout(t *testing.T) {
cc := newCancelContext()
cf := cc.buildCancelFunc()
@@ -1245,9 +1260,8 @@ func TestAgentCancelFunc_MultiCall_TimeoutEscalationReturnsErrCancelTimeout(t *t
if !ok {
t.Fatal("first should contribute")
}
- time.Sleep(50 * time.Millisecond)
- if !cc.isImmediate() {
- t.Error("should escalate to immediate")
+ if !waitUntil(t, 5*time.Second, cc.isImmediate) {
+ t.Fatal("should escalate to immediate")
}
if atomic.LoadInt32(&cc.timeoutEscalated) != 1 {
t.Error("expected timeout escalated")
@@ -1261,13 +1275,28 @@ func TestAgentCancelFunc_MultiCall_TimeoutDeadlineJoinUsesAbsoluteTime(t *testin
if !ok1 {
t.Fatal("first should contribute")
}
+ firstDeadline := atomic.LoadInt64(&cc.deadlineUnixNano)
+ if firstDeadline == 0 {
+ t.Fatal("first call must arm a deadline")
+ }
+
+ joinAt := time.Now()
_, ok2 := cf(WithCancelMode(CancelAfterChatModel), WithCancelTimeout(20*time.Millisecond))
if !ok2 {
t.Fatal("second should contribute")
}
- time.Sleep(50 * time.Millisecond)
- if !cc.isImmediate() {
- t.Error("should escalate to immediate after short timeout")
+ // The join keeps the earliest absolute deadline: a 20ms registration made
+ // right after a 200ms one must replace it, which only holds when deadlines
+ // are compared as absolute times instead of durations.
+ joined := atomic.LoadInt64(&cc.deadlineUnixNano)
+ if joined >= firstDeadline {
+ t.Fatalf("deadline = %d, first = %d; the 20ms registration must shorten the 200ms one", joined, firstDeadline)
+ }
+ if limit := joinAt.Add(50 * time.Millisecond).UnixNano(); joined > limit {
+ t.Fatalf("deadline = %d, want roughly join+20ms (<= %d)", joined, limit)
+ }
+ if !waitUntil(t, 5*time.Second, cc.isImmediate) {
+ t.Fatal("should escalate to immediate after short timeout")
}
}
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index b6eaafd08e..4dfd2d5055 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -105,6 +105,21 @@ func ProcessChunksForPipeline(
return metadata, nil
}
+// StampDatasetPagerank writes the dataset's pagerank onto every chunk a run is
+// about to index. Python's index-time doc template carries it
+// (`if task["pagerank"]: doc[PAGERANK_FLD] = int(...)`), which is what makes a
+// pagerank configured before parsing apply to the chunks it produces. Without
+// it only the chunks that already existed when the setting was saved carry the
+// value, since the dataset-level update rewrites rows rather than future ones.
+func StampDatasetPagerank(chunks []map[string]any, pagerank int64) {
+ if pagerank == 0 {
+ return
+ }
+ for _, chunk := range chunks {
+ chunk[common.PAGERANK_FLD] = pagerank
+ }
+}
+
// requireStringText enforces the pre-index wire contract: every chunk must
// carry a string "text" field before chunk-id generation or persistence mapping.
func requireStringText(ck map[string]any) (string, error) {
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index adb07e1c27..8059cd138e 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -12,6 +12,27 @@ import (
"ragflow/internal/parser/parser"
)
+// TestStampDatasetPagerank pins the index-time stamp: a configured dataset
+// pagerank lands on every chunk the run indexes, and zero leaves chunks
+// untouched (Python only stamps a truthy pagerank).
+func TestStampDatasetPagerank(t *testing.T) {
+ chunks := []map[string]any{{"id": "a"}, {"id": "b"}}
+
+ StampDatasetPagerank(chunks, 0)
+ for _, chunk := range chunks {
+ if _, ok := chunk[common.PAGERANK_FLD]; ok {
+ t.Fatalf("zero pagerank stamped a field: %#v", chunk)
+ }
+ }
+
+ StampDatasetPagerank(chunks, 7)
+ for _, chunk := range chunks {
+ if got := chunk[common.PAGERANK_FLD]; got != int64(7) {
+ t.Fatalf("pagerank_fea = %#v, want 7", got)
+ }
+ }
+}
+
func TestProcessChunksForPipeline_PreservesTypedMetadata(t *testing.T) {
want := map[string]any{
"year": float64(2025), "verified": false, "scalar": map[string]any{"x": 1},
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 84d02d0d7a..727de8f315 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -338,6 +338,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
return nil, err
}
indexChunks := append(chunks, parentChunks...)
+ indexdoc.StampDatasetPagerank(indexChunks, s.taskCtx.KB.Pagerank)
if err := s.indexWriter.Write(ctx, indexChunks); err != nil {
if cleanupErr := s.compensateFailedIndexWrite(ctx, indexChunks); cleanupErr != nil {
return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", err, cleanupErr)
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 3ee6d51d8a..b6025c5b93 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -1479,6 +1479,12 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
if tagFeas != nil {
chunkData["tag_feas"] = tagFeas
}
+ if kb.Pagerank != 0 {
+ // Python's chunk_api.create stamps the dataset's pagerank on a manually
+ // added chunk (`if kb.pagerank: d["pagerank_fea"] = kb.pagerank`); a
+ // chunk that skipped it would rank at 0 until the setting is re-saved.
+ chunkData[common.PAGERANK_FLD] = kb.Pagerank
+ }
if req.ImageBase64 != nil {
var imageBinary []byte
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index cbe742fbda..75d62fbe22 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -1159,7 +1159,7 @@ func TestAddChunkSuccess(t *testing.T) {
return permission.ErrPermissionDenied
},
getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {
- return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1", Language: &language}, nil
+ return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1", Language: &language, Pagerank: 7}, nil
},
getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {
driver := &stubEmbeddingDriver{
@@ -1210,6 +1210,12 @@ func TestAddChunkSuccess(t *testing.T) {
if resp.Chunk["content"] != "chunk body" {
t.Fatalf("content = %v, want chunk body", resp.Chunk["content"])
}
+ if len(engine.insertedChunks) != 1 {
+ t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))
+ }
+ if got := engine.insertedChunks[0][common.PAGERANK_FLD]; got != int64(7) {
+ t.Fatalf("pagerank_fea = %v, want the dataset's 7", got)
+ }
if resp.Chunk["document"] != "doc-1.txt" {
t.Fatalf("document = %v, want doc-1.txt", resp.Chunk["document"])
}
diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.go
index 76b32c574a..79217a83eb 100644
--- a/internal/service/nlp/reranker.go
+++ b/internal/service/nlp/reranker.go
@@ -1055,6 +1055,12 @@ func toFloat64(v interface{}) (float64, bool) {
return float64(val), true
case int32:
return float64(val), true
+ case string:
+ // The engine's field projection stringifies non-string values
+ // (pagerank_fea arrives as "7"), and Python's rerank coerces the same
+ // string through np.array(..., dtype=float).
+ f, err := strconv.ParseFloat(strings.TrimSpace(val), 64)
+ return f, err == nil
default:
return 0, false
}
diff --git a/internal/service/nlp/reranker_rank_feature_test.go b/internal/service/nlp/reranker_rank_feature_test.go
index 2b85150de1..37ccd3dd66 100644
--- a/internal/service/nlp/reranker_rank_feature_test.go
+++ b/internal/service/nlp/reranker_rank_feature_test.go
@@ -85,6 +85,41 @@ func TestApplyRankFeatureScores_PagerankOnly(t *testing.T) {
}
}
+// TestApplyRankFeatureScores_StringifiedPagerank pins the engine field
+// projection: GetFields stringifies non-string values, so a chunk's pagerank
+// reaches the reranker as "7" and must still count (Python coerces it through
+// np.array(..., dtype=float)).
+func TestApplyRankFeatureScores_StringifiedPagerank(t *testing.T) {
+ chunks := []map[string]interface{}{
+ {common.PAGERANK_FLD: "7"},
+ {common.PAGERANK_FLD: 3},
+ }
+ got := applyRankFeatureScores(chunks, []float64{1, 1}, nil)
+ want := []float64{8, 4}
+ if !floatsClose(got, want, 1e-9) {
+ t.Fatalf("got %v, want %v", got, want)
+ }
+}
+
+// TestApplyRankFeatureScoresForIDs_ProjectionShapes mirrors the ES field map as
+// the reranker actually receives it: numbers stringified and tag_feas rendered
+// as JSON.
+func TestApplyRankFeatureScoresForIDs_ProjectionShapes(t *testing.T) {
+ field := map[string]map[string]interface{}{
+ "a": {common.TAG_FLD: `{"tag_a":0.5}`, common.PAGERANK_FLD: "5"},
+ "b": {common.PAGERANK_FLD: "0"},
+ }
+ rankFeature := map[string]float64{"tag_a": 1}
+ got := applyRankFeatureScoresForIDs([]string{"a", "b"}, field, []float64{0.2, 0.2}, rankFeature)
+ want := []float64{
+ wantRankFeature(0.2, map[string]float64{"tag_a": 0.5}, 5, rankFeature),
+ 0.2,
+ }
+ if !floatsClose(got, want, 1e-9) {
+ t.Fatalf("got %v, want %v", got, want)
+ }
+}
+
func TestApplyRankFeatureScores_PagerankOnlyQueryNoDivZero(t *testing.T) {
chunks := []map[string]interface{}{{common.PAGERANK_FLD: 10}}
sim := []float64{0.5}
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index 29e3555519..b5260c680e 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -556,7 +556,13 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrieval
for i, id := range searchResult.IDs {
sim[i] = scores[id]
}
- return sim, make([]float64, len(sim)), sim, nil
+ // A dense-only request has no scored bool query, so the engine never
+ // applies its rank_feature clauses; the rank features have to be added
+ // here to reach the similarity at all.
+ // The boost mutates sim, so vector_similarity is sampled first.
+ rawSim := slices.Clone(sim)
+ sim = applyRankFeatureScoresForIDs(searchResult.IDs, searchResult.Field, sim, *req.RankFeature)
+ return sim, make([]float64, len(sim)), rawSim, nil
}
knnResult, err := s.docEngine.KNNScores(ctx, searchResult.Chunks, searchResult.QueryVector, len(searchResult.IDs))
@@ -728,6 +734,14 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
// child fragments to their parent, row_id() the table row identity.
"content_with_weight", "doc_type_kwd", "mom_id", "row_id()",
"compile_kwd", "scope_kwd",
+ // Generated questions: question_tks feeds the reranker's term input (×6),
+ // question_kwd is what the dataset search API reports as "questions".
+ "question_tks", "question_kwd",
+ // Rank-feature inputs. The reranker reads them off searchResult.Field,
+ // which is the engine's projection of exactly this list, so leaving them
+ // out scored every chunk at pagerank 0 and dropped the tag-feature
+ // weight entirely. Python's default src carries both.
+ common.PAGERANK_FLD, common.TAG_FLD,
"_score",
}
@@ -806,7 +820,8 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
// Build source with vector column for ES
searchSrc := make([]string, len(searchRequest.SelectFields))
copy(searchSrc, searchRequest.SelectFields)
- if s.docEngine.GetType() == string(engine.EngineElasticsearch) || engine.IsOceanBaseFamily(s.docEngine.GetType()) || engine.IsVastbase(s.docEngine.GetType()) {
+ driverType := s.docEngine.GetType()
+ if driverType == string(engine.EngineElasticsearch) || engine.IsOceanBaseFamily(driverType) || engine.IsVastbase(driverType) {
searchSrc = append(searchSrc, matchDense.VectorColumnName)
}
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index 4f512eb2c5..b8e04a27d1 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -194,6 +194,164 @@ func TestRetrievalPreservesCompilationKindWhenPresent(t *testing.T) {
}
}
+// TestRetrievalProjectsRankFeatureFields pins the source projection. The
+// reranker reads pagerank_fea and tag_feas off Field, and Field is the engine's
+// projection of exactly SelectFields, so both must be requested.
+func TestRetrievalProjectsRankFeatureFields(t *testing.T) {
+ oldQueryBuilder := globalQueryBuilder
+ globalQueryBuilder = NewQueryBuilder()
+ defer func() { globalQueryBuilder = oldQueryBuilder }()
+
+ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
+ {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},
+ }}
+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
+ threshold := 0.0
+ vectorWeight := 1.0
+ aggs := false
+ if _, err := service.Retrieval(t.Context(), &RetrievalRequest{
+ Question: "alpha",
+ TenantIDs: []string{"tenant-1"},
+ Page: 1,
+ PageSize: 1,
+ SimilarityThreshold: &threshold,
+ VectorSimilarityWeight: &vectorWeight,
+ Aggs: &aggs,
+ }); err != nil {
+ t.Fatalf("Retrieval failed: %v", err)
+ }
+ if len(docEngine.selectedFields) == 0 {
+ t.Fatal("no search was issued")
+ }
+ for _, fields := range docEngine.selectedFields {
+ if !slices.Contains(fields, common.PAGERANK_FLD) || !slices.Contains(fields, common.TAG_FLD) {
+ t.Fatalf("search fields = %v, want %s and %s", fields, common.PAGERANK_FLD, common.TAG_FLD)
+ }
+ }
+}
+
+// TestRetrievalProjectsQuestionTokenFields pins the same projection contract for
+// the generated-question fields. The reranker folds question_tks into its term
+// input with a ×6 weight and the dataset search API reports question_kwd as
+// "questions", but both are read off Field — the engine's projection of exactly
+// SelectFields — so omitting them scores those tokens as absent.
+func TestRetrievalProjectsQuestionTokenFields(t *testing.T) {
+ oldQueryBuilder := globalQueryBuilder
+ globalQueryBuilder = NewQueryBuilder()
+ defer func() { globalQueryBuilder = oldQueryBuilder }()
+
+ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
+ {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "question_tks": "beta", "_score": 0.9},
+ }}
+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
+ threshold := 0.0
+ vectorWeight := 1.0
+ aggs := false
+ if _, err := service.Retrieval(t.Context(), &RetrievalRequest{
+ Question: "alpha",
+ TenantIDs: []string{"tenant-1"},
+ Page: 1,
+ PageSize: 1,
+ SimilarityThreshold: &threshold,
+ VectorSimilarityWeight: &vectorWeight,
+ Aggs: &aggs,
+ }); err != nil {
+ t.Fatalf("Retrieval failed: %v", err)
+ }
+ if len(docEngine.selectedFields) == 0 {
+ t.Fatal("no search was issued")
+ }
+ for _, fields := range docEngine.selectedFields {
+ if !slices.Contains(fields, "question_tks") || !slices.Contains(fields, "question_kwd") {
+ t.Fatalf("search fields = %v, want question_tks and question_kwd", fields)
+ }
+ }
+}
+
+// TestRetrievalPagerankBoostsRanking drives the whole path: identical chunks,
+// only pagerank_fea differs, so the boosted chunk must win and its similarity
+// must carry the pagerank on top of the hybrid score.
+func TestRetrievalPagerankBoostsRanking(t *testing.T) {
+ oldQueryBuilder := globalQueryBuilder
+ globalQueryBuilder = NewQueryBuilder()
+ defer func() { globalQueryBuilder = oldQueryBuilder }()
+
+ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
+ {"id": "plain", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5},
+ {"id": "boosted", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(4)},
+ }}
+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
+ threshold := 0.0
+ vectorWeight := 1.0
+ aggs := false
+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{
+ Question: "alpha",
+ TenantIDs: []string{"tenant-1"},
+ Page: 1,
+ PageSize: 2,
+ SimilarityThreshold: &threshold,
+ VectorSimilarityWeight: &vectorWeight,
+ Aggs: &aggs,
+ })
+ if err != nil {
+ t.Fatalf("Retrieval failed: %v", err)
+ }
+ if len(result.Chunks) != 2 {
+ t.Fatalf("chunks = %d, want 2", len(result.Chunks))
+ }
+ if got := result.Chunks[0]["chunk_id"]; got != "boosted" {
+ t.Fatalf("first chunk = %v, want boosted (pagerank must outrank the tie)", got)
+ }
+ boosted, _ := result.Chunks[0]["similarity"].(float64)
+ plain, _ := result.Chunks[1]["similarity"].(float64)
+ if boosted-plain < 4-1e-9 {
+ t.Fatalf("similarity delta = %v, want the 4-point pagerank boost", boosted-plain)
+ }
+}
+
+// TestRetrievalVectorOnlyKeepsPagerank covers the dense-only path: the engine
+// issues no scored bool query there, so the engine can never apply its
+// rank_feature clause and the similarity must pick pagerank up locally.
+// vector_similarity keeps the engine's raw kNN score (asserted below).
+func TestRetrievalVectorOnlyKeepsPagerank(t *testing.T) {
+ oldQueryBuilder := globalQueryBuilder
+ globalQueryBuilder = NewQueryBuilder()
+ defer func() { globalQueryBuilder = oldQueryBuilder }()
+
+ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
+ {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(3)},
+ }}
+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
+ threshold := 0.0
+ vectorWeight := 1.0
+ aggs := false
+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{
+ Question: "alpha",
+ TenantIDs: []string{"tenant-1"},
+ Page: 1,
+ PageSize: 1,
+ SimilarityThreshold: &threshold,
+ VectorSimilarityWeight: &vectorWeight,
+ Aggs: &aggs,
+ VectorOnly: true,
+ })
+ if err != nil {
+ t.Fatalf("Retrieval failed: %v", err)
+ }
+ if len(result.Chunks) != 1 {
+ t.Fatalf("chunks = %d, want 1", len(result.Chunks))
+ }
+ if got, _ := result.Chunks[0]["similarity"].(float64); got != 3.5 {
+ t.Fatalf("similarity = %v, want engine score 0.5 + pagerank 3", got)
+ }
+ if got, _ := result.Chunks[0]["vector_similarity"].(float64); got != 0.5 {
+ t.Fatalf("vector_similarity = %v, want the raw kNN score 0.5 (rank features belong to similarity only)", got)
+ }
+ if got, _ := result.Chunks[0]["term_similarity"].(float64); got != 0 {
+ t.Fatalf("term_similarity = %v, want 0 for a dense-only request", got)
+ }
+}
+
func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()