RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(retrieval): align pagerank and rank-feature scoring (#21542)
qinling02107c8a7f26a148
diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.goindex 1a761d7554..a911617af1 100644--- a/internal/agent/tool/retrieval_nlp.go+++ b/internal/agent/tool/retrieval_nlp.go@@ -256,19 +256,7 @@ func (a *NLPRetrievalAdapter) Search(ctx context.Context, db *gorm.DB, req Retri}query = retrievalUserPrefixPattern.ReplaceAllString(query, "")vectorOnly := req.KeywordsSimilarityWeight != nil && *req.KeywordsSimilarityWeight < 0.01- // rank_feature (Python retrieve: rank_feature=label_question(question,- // self.kbs)). Prefer a feature supplied on the request (computed by RAGTools- // from its own KB objects) so the agentic tool stays authoritative; fall- // back to the enhancer, which resolves the KB objects itself.- var rankFeature map[string]float64- if req.RankFeature != nil && len(*req.RankFeature) > 0 {- rankFeature = *req.RankFeature- } else if a.enhancer != nil && !vectorOnly {- // Rank features are extra scoring clauses on the TEXT leg; a- // vector-only request has no text leg, so labelling the question would- // be a tag lookup whose result nothing consumes.- rankFeature = a.enhancer.LabelQuestion(ctx, query, datasets.kbs)- }+ rankFeature := resolveRankFeature(ctx, req.RankFeature, a.enhancer, query, datasets.kbs, vectorOnly)rerankModel, err := a.resolveRerankModel(ctx, req, datasets.kbs[0].TenantID)if err != nil {return nil, err@@ -280,9 +268,7 @@ func (a *NLPRetrievalAdapter) Search(ctx context.Context, db *gorm.DB, req RetrinlpReq := nlpRequestFromRetrieval(preparedReq, datasets.tenantIDs, topN, embeddingModel, preparedReq.ExcludeCompiled)nlpReq.Language = entity.KnowledgebasesLanguage(datasets.kbs)nlpReq.RerankModel = rerankModel- if rankFeature != nil {- nlpReq.RankFeature = &rankFeature- }+ nlpReq.RankFeature = &rankFeatureres, err := a.svc.Retrieval(ctx, nlpReq)if err != nil {@@ -302,6 +288,46 @@ func (a *NLPRetrievalAdapter) Search(ctx context.Context, db *gorm.DB, req Retrireturn out, nil}+// resolveRankFeature resolves the rank features handed to the nlp layer, which+// takes them as *map and distinguishes "unset" from "empty". The result is never+// nil: a nil map leaves that pointer unset and the nlp layer substitutes its+// {pagerank_fea: 10} default.+func resolveRankFeature(+ ctx context.Context,+ requested *map[string]float64,+ enhancer retrievalEnhancer,+ query string,+ kbs []*entity.Knowledgebase,+ vectorOnly bool,+) map[string]float64 {+ // A caller-supplied feature (computed by RAGTools from its own KB objects) is+ // authoritative: nil means "said nothing, label it", an empty map means+ // "there are none" and must not turn into a label lookup.+ if requested != nil {+ return cloneRankFeature(*requested)+ }+ // Only the engine-side clause needs a text leg, so a dense-only request is not+ // labelled. The features still reach the similarity - with no scored bool, the+ // nlp layer applies them locally - so a caller-supplied feature keeps its+ // effect in vector-only mode and only the enhancer fallback is skipped.+ if enhancer == nil || vectorOnly {+ return map[string]float64{}+ }+ if labels := enhancer.LabelQuestion(ctx, query, kbs); labels != nil {+ return cloneRankFeature(labels)+ }+ return map[string]float64{}+}++// cloneRankFeature detaches the request's map from the caller's.+func cloneRankFeature(features map[string]float64) map[string]float64 {+ clone := make(map[string]float64, len(features))+ for field, weight := range features {+ clone[field] = weight+ }+ return clone+}+func nlpRequestFromRetrieval(req RetrievalRequest,tenantIDs []string,@@ -365,17 +391,6 @@ func nlpRequestFromRetrieval("must_not": map[string]interface{}{"exists": "compile_kwd"},}}- // Restrict to ordinary document text chunks with the SAME filter grep_chunks- // applies: available_int=1 (must_not available_int<1 semantics so chunks- // whose available_int is absent still pass) and must_not exists compile_kwd- // to exclude knowledge-compiled products. Setting available_int explicitly- // here matches grep exactly (both go through buildBoolQueryFromCondition).- if req.OnlyOriginalText {- nlpReq.Filter = map[string]interface{}{- "available_int": 1,- "must_not": map[string]interface{}{"exists": "compile_kwd"},- }- }return nlpReq}diff --git a/internal/agent/tool/retrieval_nlp_test.go b/internal/agent/tool/retrieval_nlp_test.goindex 40e14953dd..e9dc03b4ca 100644--- a/internal/agent/tool/retrieval_nlp_test.go+++ b/internal/agent/tool/retrieval_nlp_test.go@@ -20,9 +20,10 @@// nlp.RetrievalService, which requires a doc engine + document// DAO. Those require network access and aren't worth standing up// in a unit test. Instead we exercise the translation layer-// (translateChunk + helpers) directly — that's the surface that+// (translateChunk + helpers) and the rank-feature resolution+// (resolveRankFeature) directly — that's the surface that// differs between the nlp chunk shape and the agent-tool chunk-// shape, and it's the only piece the adapter owns.+// shape, and the only piece the adapter itself decides.package tool@@ -626,3 +627,142 @@ func (f fakeKnowledgebaseLookup) GetByName(_ context.Context, _ *gorm.DB, name,}return nil, gorm.ErrRecordNotFound}++// stubRankFeatureEnhancer records whether the question was labelled.+type stubRankFeatureEnhancer struct {+ labels map[string]float64+ calls int+}++func (s *stubRankFeatureEnhancer) CrossLanguages(context.Context, string, string, []string) (string, error) {+ return "", nil+}++func (s *stubRankFeatureEnhancer) FilterDocuments(+ context.Context, map[string]any, string, *modelModule.ChatModel, []string, []string,+) ([]string, error) {+ return nil, nil+}++func (s *stubRankFeatureEnhancer) LabelQuestion(+ context.Context, string, []*entity.Knowledgebase,+) map[string]float64 {+ s.calls+++ return s.labels+}++func (s *stubRankFeatureEnhancer) RetrieveByChildren(+ context.Context, []map[string]any, []string,+) []map[string]any {+ return nil+}++func testRankFeatureKBs() []*entity.Knowledgebase {+ return []*entity.Knowledgebase{{ID: "kb-1", TenantID: "tenant-1"}}+}++// No tag source: labelling yields nil, which must still reach the request as an+// empty map instead of letting the nlp layer apply its default.+func TestResolveRankFeature_NoTagSourceReturnsEmptyMap(t *testing.T) {+ enhancer := &stubRankFeatureEnhancer{labels: nil}++ got := resolveRankFeature(context.Background(), nil, enhancer, "q", testRankFeatureKBs(), false)++ if got == nil {+ t.Fatal("rankFeature = nil, want an empty (non-nil) map")+ }+ if len(got) != 0 {+ t.Fatalf("rankFeature = %v, want empty", got)+ }+ if enhancer.calls != 1 {+ t.Fatalf("LabelQuestion calls = %d, want 1", enhancer.calls)+ }+}++func TestResolveRankFeature_TagMatchIsForwarded(t *testing.T) {+ enhancer := &stubRankFeatureEnhancer{labels: map[string]float64{"tag_1": 0.8}}++ got := resolveRankFeature(context.Background(), nil, enhancer, "q", testRankFeatureKBs(), false)++ if got["tag_1"] != 0.8 {+ t.Fatalf("rankFeature = %v, want tag_1=0.8", got)+ }+}++// A caller-supplied feature wins over the enhancer and must not trigger a label+// lookup.+func TestResolveRankFeature_PrefersRequestSuppliedFeature(t *testing.T) {+ enhancer := &stubRankFeatureEnhancer{labels: map[string]float64{"from_enhancer": 0.1}}+ requested := map[string]float64{"from_request": 0.5}++ got := resolveRankFeature(context.Background(), &requested, enhancer, "q", testRankFeatureKBs(), false)++ if got["from_request"] != 0.5 || len(got) != 1 {+ t.Fatalf("rankFeature = %v, want only from_request=0.5", got)+ }+ if enhancer.calls != 0 {+ t.Fatalf("LabelQuestion calls = %d, want 0", enhancer.calls)+ }+}++// A vector-only request skips the label lookup; the map must still be non-nil.+func TestResolveRankFeature_VectorOnlySkipsLabelling(t *testing.T) {+ enhancer := &stubRankFeatureEnhancer{labels: map[string]float64{"tag_1": 0.8}}++ got := resolveRankFeature(context.Background(), nil, enhancer, "q", testRankFeatureKBs(), true)++ if got == nil {+ t.Fatal("rankFeature = nil, want an empty (non-nil) map")+ }+ if len(got) != 0 {+ t.Fatalf("rankFeature = %v, want empty", got)+ }+ if enhancer.calls != 0 {+ t.Fatalf("LabelQuestion calls = %d, want 0", enhancer.calls)+ }+}++func TestResolveRankFeature_NilEnhancerReturnsEmptyMap(t *testing.T) {+ got := resolveRankFeature(context.Background(), nil, nil, "q", testRankFeatureKBs(), false)++ if got == nil {+ t.Fatal("rankFeature = nil, want an empty (non-nil) map")+ }+ if len(got) != 0 {+ t.Fatalf("rankFeature = %v, want empty", got)+ }+}++// nil and {} are distinct: nil asks for labelling, {} means there are none.+func TestResolveRankFeature_ExplicitEmptyMapSuppressesLabelling(t *testing.T) {+ enhancer := &stubRankFeatureEnhancer{labels: map[string]float64{"tag_1": 0.8}}+ requested := map[string]float64{}++ got := resolveRankFeature(context.Background(), &requested, enhancer, "q", testRankFeatureKBs(), false)++ if got == nil {+ t.Fatal("rankFeature = nil, want an empty (non-nil) map")+ }+ if len(got) != 0 {+ t.Fatalf("rankFeature = %v, want empty", got)+ }+ if enhancer.calls != 0 {+ t.Fatalf("LabelQuestion calls = %d, want 0", enhancer.calls)+ }+}++// The result must not alias the caller's map.+func TestResolveRankFeature_ReturnsDetachedCopy(t *testing.T) {+ requested := map[string]float64{"tag_1": 0.4}++ got := resolveRankFeature(context.Background(), &requested, nil, "q", testRankFeatureKBs(), false)+ got["tag_1"] = 0.9+ got["injected"] = 1.0++ if requested["tag_1"] != 0.4 {+ t.Fatalf("caller map mutated through the result: %v", requested)+ }+ if _, ok := requested["injected"]; ok {+ t.Fatalf("caller map gained a key through the result: %v", requested)+ }+}diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex 6034e9d49c..f27dfef5ae 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -1318,7 +1318,7 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S"k": k,"num_candidates": numCandidates,"similarity": similarity,- "filter": boolQuery,+ "filter": cloneFilterSnapshot(boolQuery),}queryBody["knn"] = knnQuery@@ -1345,17 +1345,13 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.Sif rankFeatureQuery != nil {if boolQuery, ok := queryBody["query"].(map[string]interface{}); ok {if boolMap, ok := boolQuery["bool"].(map[string]interface{}); ok {- if should, ok := boolMap["should"].([]interface{}); ok {- for _, q := range rankFeatureQuery {- boolMap["should"] = append(should, q)- }- } else {- interfaceSlice := make([]interface{}, len(rankFeatureQuery))- for i, q := range rankFeatureQuery {- interfaceSlice[i] = q- }- boolMap["should"] = interfaceSlice+ // Accumulate into a local slice first: re-appending to the+ // pre-loop header keeps only the last rank_feature clause.+ should, _ := boolMap["should"].([]interface{})+ for _, q := range rankFeatureQuery {+ should = append(should, q)}+ boolMap["should"] = should}}}@@ -1497,15 +1493,10 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S// back empty here, which flattened every hybrid chunk's score to 0.scoreColumn := "_score"- pagerankField := common.PAGERANK_FLD- if isSkillIndex {- pagerankField = ""- }- if isMemoryIndex {- pagerankField = ""- }-- allResults = calculateScores(allResults, scoreColumn, pagerankField)+ // pagerank reaches the ES hit score only through the rank_feature query+ // built above, and reaches the final similarity in the reranker. Adding+ // it to _score here as well would count it twice.+ allResults = calculateScores(allResults, scoreColumn, "")allResults = sortByScore(allResults, limit)}@@ -2145,6 +2136,35 @@ func memoryMessageStatusBool(value interface{}) bool {}}+// cloneFilterSnapshot freezes the bool query's shape at the point the kNN clause+// is built. The reference does the same by passing bool_query.to_dict() as the+// kNN filter; the rank_feature clauses are appended to the same bool afterwards+// and belong to the scored query only.+func cloneFilterSnapshot(q map[string]interface{}) map[string]interface{} {+ if q == nil {+ return nil+ }+ clone := make(map[string]interface{}, len(q))+ for k, v := range q {+ clone[k] = v+ }+ inner, ok := q["bool"].(map[string]interface{})+ if !ok {+ return clone+ }+ innerClone := make(map[string]interface{}, len(inner))+ for k, v := range inner {+ innerClone[k] = v+ }+ for _, key := range []string{"must", "should", "filter", "must_not"} {+ if clauses, ok := inner[key].([]interface{}); ok {+ innerClone[key] = slices.Clone(clauses)+ }+ }+ clone["bool"] = innerClone+ return clone+}+// buildBoolQueryFromCondition builds an ES bool query from condition map.// Skill indexes use status, regular chunk indexes use kb_id, and memory// message indexes use memory_id plus message-specific storage fields.@@ -2156,7 +2176,6 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strinvar mustClauses []interface{}var mustNotClauses []interface{}var filterClauses []interface{}- var shouldClauses []interface{}// Memory message indexes use memory_id, regular chunk indexes use kb_id.if kbIDs != nil && len(kbIDs) > 0 {@@ -2279,36 +2298,26 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strincontinue}if k == "id" {- if v == nil || v == "" {- continue- }- if listVal, ok := v.([]interface{}); ok && len(listVal) > 0 {- shouldClauses = append(shouldClauses,- map[string]interface{}{"terms": map[string]interface{}{"id": listVal}},- map[string]interface{}{"terms": map[string]interface{}{"_id": listVal}},- )- } else if strListVal, ok := v.([]string); ok && len(strListVal) > 0 {- // A typed []string MUST be handled here: the generic loop- // below skips the "id" key, so without this branch a caller- // passing map[string]interface{}{"id": []string{...}} builds a- // query with NO id filter and the scoped read silently- // degenerates into "fetch up to Limit chunks of the document"- // (observed via list_chunks: an 11-chunk window returned 3.8MB- // because the whole 4171-chunk document was fetched instead).- shouldClauses = append(shouldClauses,- map[string]interface{}{"terms": map[string]interface{}{"id": strListVal}},- map[string]interface{}{"terms": map[string]interface{}{"_id": strListVal}},- )- } else if strVal, ok := v.(string); ok && strVal != "" {- shouldClauses = append(shouldClauses,- map[string]interface{}{"term": map[string]interface{}{"id": strVal}},- map[string]interface{}{"term": map[string]interface{}{"_id": strVal}},- )- } else if intVal, ok := v.(int); ok && intVal != 0 {- shouldClauses = append(shouldClauses,- map[string]interface{}{"term": map[string]interface{}{"id": intVal}},- map[string]interface{}{"term": map[string]interface{}{"_id": intVal}},- )+ // id is a scope restriction, never a scoring hint. It has to be a+ // mandatory clause: encoding it in bool.should relied on+ // minimum_should_match to make it binding, and Search() appends+ // rank_feature clauses to that same top-level should list — every+ // document carrying the rank feature then satisfied the id scope,+ // so a chunk-id-scoped read silently returned unrelated chunks.+ //+ // The generic loop below skips the "id" key, so a value shape this+ // helper does not understand leaves the query without any id+ // filter: list_chunks then degenerates into "fetch up to Limit+ // chunks of the document" (an 11-chunk window returned 3.8MB+ // instead of ~17KB). idTermClauses therefore accepts every list+ // flavour, not only []interface{}.+ if idClauses := idTermClauses(v); len(idClauses) > 0 {+ filterClauses = append(filterClauses, map[string]interface{}{+ "bool": map[string]interface{}{+ "should": idClauses,+ "minimum_should_match": 1,+ },+ })}continue}@@ -2361,10 +2370,6 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strinif len(filterClauses) > 0 {boolQuery["filter"] = filterClauses}- if len(shouldClauses) > 0 {- boolQuery["should"] = shouldClauses- boolQuery["minimum_should_match"] = 1- }if len(boolQuery) == 0 {return nil@@ -2373,6 +2378,43 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strinreturn map[string]interface{}{"bool": boolQuery}}+// idTermClauses renders one id condition as id/_id alternatives, so a caller+// matches whichever field it addressed: the stored id field or the+// Elasticsearch _id. The caller makes the result mandatory.+//+// The query shapes mirror the Python reference (rag/utils/es_conn.py): a+// scalar id is a term, a list of ids is a terms query.+func idTermClauses(value interface{}) []interface{} {+ switch val := value.(type) {+ case nil:+ return nil+ case string:+ if val == "" {+ return nil+ }+ return []interface{}{+ map[string]interface{}{"term": map[string]interface{}{"id": val}},+ map[string]interface{}{"term": map[string]interface{}{"_id": val}},+ }+ case int:+ if val == 0 {+ return nil+ }+ return []interface{}{+ map[string]interface{}{"term": map[string]interface{}{"id": val}},+ map[string]interface{}{"term": map[string]interface{}{"_id": val}},+ }+ }+ items, ok := conditionTermsList(value)+ if !ok || len(items) == 0 {+ return nil+ }+ return []interface{}{+ map[string]interface{}{"terms": map[string]interface{}{"id": items}},+ map[string]interface{}{"terms": map[string]interface{}{"_id": items}},+ }+}+// buildQueryStringQuery builds a query_string query from MatchTextExpr// When isSkillIndex is true, uses skill-specific fields (name_tks, tags_tks, etc.)// Otherwise uses document fields (title_tks, content_ltks, etc.)@@ -2470,14 +2512,16 @@ func buildRankFeatureQuery(rankFeature map[string]float64) []map[string]interfacvar queries []map[string]interface{}for _, fld := range keys {+ sc := rankFeature[fld]+ field := fmt.Sprintf("%s.%s", common.TAG_FLD, fld)if fld == common.PAGERANK_FLD {- continue+ // pagerank_fea is a top-level rank_feature field (the "*_fea" dynamic+ // template), not one key inside the tag_feas map.+ field = common.PAGERANK_FLD}- sc := rankFeature[fld]- tagField := fmt.Sprintf("%s.%s", common.TAG_FLD, fld)queries = append(queries, map[string]interface{}{"rank_feature": map[string]interface{}{- "field": tagField,+ "field": field,"linear": map[string]interface{}{},"boost": sc,},@@ -2642,6 +2686,17 @@ func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) mapcontinue}+ if mapVal, ok := val.(map[string]interface{}); ok {+ // Nested objects (tag_feas is a "rank_features" object) cannot+ // take the string conversion below: "%v" renders map[k:v], which+ // no rank-feature parser accepts. JSON is the string form the+ // reranker parses.+ if raw, err := json.Marshal(mapVal); err == nil {+ m[field] = string(raw)+ continue+ }+ }+if listVal, ok := val.([]interface{}); ok {if len(listVal) == 1 {if _, isArray := listVal[0].([]interface{}); !isArray {diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex 4f2b38935a..ae5b3f89a2 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -545,6 +545,62 @@ func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) {}}+// TestBuildRankFeatureQueryUsesTopLevelPagerank pins the field naming: tag+// weights address keys inside the tag_feas map, while pagerank addresses the+// top-level pagerank_fea rank_feature field.+func TestBuildRankFeatureQueryUsesTopLevelPagerank(t *testing.T) {+ queries := buildRankFeatureQuery(map[string]float64{+ common.PAGERANK_FLD: 10,+ "tag_a": 0.8,+ })+ if len(queries) != 2 {+ t.Fatalf("queries = %d, want 2", len(queries))+ }+ byField := make(map[string]map[string]interface{}, len(queries))+ for _, q := range queries {+ clause, ok := q["rank_feature"].(map[string]interface{})+ if !ok {+ t.Fatalf("not a rank_feature clause: %#v", q)+ }+ field, _ := clause["field"].(string)+ byField[field] = clause+ }+ pagerank, ok := byField[common.PAGERANK_FLD]+ if !ok {+ t.Fatalf("pagerank clause missing, got fields %#v", byField)+ }+ if got := pagerank["boost"]; got != float64(10) {+ t.Fatalf("pagerank boost = %v, want 10", got)+ }+ tagField := common.TAG_FLD + ".tag_a"+ if _, ok := byField[tagField]; !ok {+ t.Fatalf("tag clause %q missing, got fields %#v", tagField, byField)+ }+}++// TestElasticsearchGetFieldsEncodesNestedObjects covers the rank-feature+// fields: tag_feas is a "rank_features" object in _source, so the projection+// has to render it as JSON (the reranker parses JSON, not Go's "%v" map form),+// while numeric fields keep the stringified form the rest of the projection uses.+func TestElasticsearchGetFieldsEncodesNestedObjects(t *testing.T) {+ engine := &Engine{}+ chunks := []map[string]interface{}{+ {+ "id": "chunk-1",+ "tag_feas": map[string]interface{}{"tag_a": float64(0.5)},+ "pagerank_fea": float64(7),+ "content_with_weight": "body",+ },+ }++ fieldMap, ok := engine.GetFields(chunks, []string{"id", "tag_feas", "pagerank_fea"})["chunk-1"]+ if !ok {+ t.Fatalf("GetFields lost chunk-1")+ }+ assertEqual(t, fieldMap["tag_feas"], `{"tag_a":0.5}`)+ assertEqual(t, fieldMap["pagerank_fea"], "7")+}+func TestElasticsearchGetFieldsEmptyAndSkippedIDs(t *testing.T) {engine := &Engine{}@@ -760,6 +816,187 @@ func TestMapMemoryMessageToESDoc(t *testing.T) {}}+// TestSearchKeepsRankFeatureOutOfKNNFilter pins the body-level split: the+// rank_feature clauses belong to the scored query, while knn.filter is a snapshot+// of the bool taken when the kNN clause is built and must keep only the scope and+// lexical predicates.+func TestSearchKeepsRankFeatureOutOfKNNFilter(t *testing.T) {+ if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {+ t.Fatalf("init logger: %v", err)+ }+ var searchQuery map[string]interface{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if err := json.NewDecoder(r.Body).Decode(&searchQuery); err != nil {+ t.Errorf("decode search query: %v", err)+ }+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"hits":{"total":{"value":0},"hits":[]}}`))+ }))+ defer server.Close()++ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ _, err = engine.Search(t.Context(), &types.SearchRequest{+ IndexNames: []string{"ragflow_tenant"},+ KbIDs: []string{"kb-1"},+ Limit: 10,+ Filter: map[string]interface{}{"available_int": 1},+ MatchExprs: []interface{}{+ &types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},+ &types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},+ &types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},+ },+ RankFeature: map[string]float64{"pagerank_fea": 10},+ })+ if err != nil {+ t.Fatalf("Search: %v", err)+ }++ queryJSON, err := json.Marshal(searchQuery["query"])+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(queryJSON), "rank_feature") || !strings.Contains(string(queryJSON), "pagerank_fea") {+ t.Fatalf("scored query lost its rank_feature clause: %s", queryJSON)+ }++ knn, ok := searchQuery["knn"].(map[string]interface{})+ if !ok {+ t.Fatalf("KNN query missing from %#v", searchQuery)+ }+ filterJSON, err := json.Marshal(knn["filter"])+ if err != nil {+ t.Fatal(err)+ }+ if strings.Contains(string(filterJSON), "rank_feature") {+ t.Fatalf("knn.filter carries rank_feature clauses: %s", filterJSON)+ }+ // The snapshot must still be the scope + lexical predicate, not an empty one.+ if !strings.Contains(string(filterJSON), "query_string") {+ t.Fatalf("knn.filter lost the lexical predicate: %s", filterJSON)+ }+}++// TestSearchKeepsEveryRankFeatureClause covers the multi-feature case: appending+// to the pre-loop should header used to keep only the last clause.+func TestSearchKeepsEveryRankFeatureClause(t *testing.T) {+ if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {+ t.Fatalf("init logger: %v", err)+ }+ var searchQuery map[string]interface{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if err := json.NewDecoder(r.Body).Decode(&searchQuery); err != nil {+ t.Errorf("decode search query: %v", err)+ }+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"hits":{"total":{"value":0},"hits":[]}}`))+ }))+ defer server.Close()++ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ _, err = engine.Search(t.Context(), &types.SearchRequest{+ IndexNames: []string{"ragflow_tenant"},+ KbIDs: []string{"kb-1"},+ Limit: 10,+ Filter: map[string]interface{}{"available_int": 1},+ MatchExprs: []interface{}{+ &types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},+ &types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},+ &types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},+ },+ RankFeature: map[string]float64{"tag_1": 0.8, "tag_2": 0.5},+ })+ if err != nil {+ t.Fatalf("Search: %v", err)+ }++ query, ok := searchQuery["query"].(map[string]interface{})+ if !ok {+ t.Fatalf("query missing from %#v", searchQuery)+ }+ boolMap, ok := query["bool"].(map[string]interface{})+ if !ok {+ t.Fatalf("bool query missing from %#v", query)+ }+ should, _ := boolMap["should"].([]interface{})+ var fields []string+ for _, clause := range should {+ clauseMap, ok := clause.(map[string]interface{})+ if !ok {+ continue+ }+ rankFeature, ok := clauseMap["rank_feature"].(map[string]interface{})+ if !ok {+ continue+ }+ if field, ok := rankFeature["field"].(string); ok {+ fields = append(fields, field)+ }+ }+ if len(fields) != 2 {+ t.Fatalf("rank_feature clauses = %v, want both tag fields", fields)+ }+ joined := strings.Join(fields, ",")+ if !strings.Contains(joined, "tag_feas.tag_1") || !strings.Contains(joined, "tag_feas.tag_2") {+ t.Fatalf("rank_feature fields = %v, want tag_feas.tag_1 and tag_feas.tag_2", fields)+ }+}++// TestSearchDoesNotAddPageRankToHitScore pins the hybrid score contract: _score is+// the ES hit score and nothing else. pagerank already reaches the hit score+// through the rank_feature clause and the final similarity through the reranker;+// folding it in here again would count it twice.+func TestSearchDoesNotAddPageRankToHitScore(t *testing.T) {+ if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {+ t.Fatalf("init logger: %v", err)+ }+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"hits":{"total":{"value":1},"hits":[{"_id":"chunk-1","_index":"ragflow_tenant","_score":0.87,"_source":{"content_with_weight":"alpha","pagerank_fea":69,"tag_feas":{"tag_1":0.5}}}]}}`))+ }))+ defer server.Close()++ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ res, err := engine.Search(t.Context(), &types.SearchRequest{+ IndexNames: []string{"ragflow_tenant"},+ KbIDs: []string{"kb-1"},+ Limit: 10,+ Filter: map[string]interface{}{"available_int": 1},+ MatchExprs: []interface{}{+ &types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},+ &types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},+ &types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},+ },+ RankFeature: map[string]float64{"pagerank_fea": 10},+ })+ if err != nil {+ t.Fatalf("Search: %v", err)+ }+ if len(res.Chunks) != 1 {+ t.Fatalf("Search returned %d chunks, want 1", len(res.Chunks))+ }+ if got, ok := res.Chunks[0]["_score"].(float64); !ok || got != 0.87 {+ t.Fatalf("_score = %#v, want the hit score 0.87 (pagerank must not be folded in)", res.Chunks[0]["_score"])+ }+ if got, ok := res.Chunks[0]["pagerank_fea"].(float64); !ok || got != 69 {+ t.Fatalf("pagerank_fea = %#v, want 69 preserved for the reranker", res.Chunks[0]["pagerank_fea"])+ }+}+// TestMapMemoryMessageESUpdateFieldsRefreshesTokens: writing content via// the update path recomputes tokenized_content_ltks before write, same// as the Python update path.@@ -776,3 +1013,50 @@ func TestMapMemoryMessageESUpdateFieldsRefreshesTokens(t *testing.T) {}assertEqual(t, doc["status_int"], 0)}++func TestCloneFilterSnapshotIsolatesLaterShouldAppends(t *testing.T) {+ boolMap := map[string]interface{}{+ "must": []interface{}{map[string]interface{}{"term": map[string]interface{}{"kb_id": "kb-1"}}},+ "should": []interface{}{map[string]interface{}{"match_all": map[string]interface{}{}}},+ }+ boolQuery := map[string]interface{}{"bool": boolMap}++ snapshot := cloneFilterSnapshot(boolQuery)++ // Mirror what the rank_feature block does after the kNN clause is built: it+ // appends to the scored query's bool/should.+ boolMap["should"] = append(+ boolMap["should"].([]interface{}),+ map[string]interface{}{"rank_feature": map[string]interface{}{"field": "pagerank_fea", "boost": 10.0}},+ )++ if got := len(boolMap["should"].([]interface{})); got != 2 {+ t.Fatalf("original should has %d clauses, want 2", got)+ }+ snapshotBool, ok := snapshot["bool"].(map[string]interface{})+ if !ok {+ t.Fatalf("snapshot bool missing from %#v", snapshot)+ }+ if got := len(snapshotBool["should"].([]interface{})); got != 1 {+ t.Fatalf("snapshot should has %d clauses, want 1 (rank_feature must stay out of knn.filter)", got)+ }+ if got := len(snapshotBool["must"].([]interface{})); got != 1 {+ t.Fatalf("snapshot must has %d clauses, want 1", got)+ }+}++func TestCloneFilterSnapshotNil(t *testing.T) {+ if got := cloneFilterSnapshot(nil); got != nil {+ t.Fatalf("cloneFilterSnapshot(nil) = %#v, want nil", got)+ }+}++func TestCloneFilterSnapshotNonBoolQuery(t *testing.T) {+ query := map[string]interface{}{+ "query_string": map[string]interface{}{"query": "alpha"},+ }+ snapshot := cloneFilterSnapshot(query)+ if snapshot["query_string"] == nil {+ t.Fatalf("snapshot = %#v, want the query_string clause preserved", snapshot)+ }+}diff --git a/internal/engine/elasticsearch/chunk_test.go b/internal/engine/elasticsearch/chunk_test.goindex a4c5ca2837..e5f0c627a0 100644--- a/internal/engine/elasticsearch/chunk_test.go+++ b/internal/engine/elasticsearch/chunk_test.go@@ -20,9 +20,12 @@ import ("bytes""context""encoding/json"+ "net/http"+ "net/http/httptest""strings""testing"+ "github.com/elastic/go-elasticsearch/v8""ragflow/internal/common""ragflow/internal/engine/types")@@ -397,17 +400,45 @@ func TestBuildBoolQueryFromConditionUsesLegacyGraphFallback(t *testing.T) {// `should: [{...}, {...}]`. ES rejects the nested form as malformed.// This test pins down the flat shape for the list, string, and int// branches.+//+// The id restriction must also stay out of the top-level bool.should:+// Search appends rank_feature clauses to that slot, and a top-level+// minimum_should_match=1 then let any rank-feature-carrying document satisfy+// the id scope. id is emitted as a mandatory filter clause whose nested bool+// keeps the id/_id alternative, exactly as the Python reference does+// (rag/utils/es_conn.py: scalar id -> term, list -> terms).func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {- check := func(name string, cond map[string]interface{}, wantFields []string) {+ check := func(name string, cond map[string]interface{}, wantFields []string, wantOp string) {t.Helper()got := buildBoolQueryFromCondition(cond, nil, false, false)outer, ok := got["bool"].(map[string]interface{})if !ok {t.Fatalf("%s: missing bool wrapper: %v", name, got)}- should, ok := outer["should"].([]interface{})+ if _, exists := outer["should"]; exists {+ t.Fatalf("%s: id leaked into the top-level should slot: %v", name, outer["should"])+ }+ if _, exists := outer["minimum_should_match"]; exists {+ t.Fatalf("%s: top-level minimum_should_match would also bind rank_feature clauses", name)+ }+ filter, ok := outer["filter"].([]interface{})+ if !ok || len(filter) != 1 {+ t.Fatalf("%s: filter = %v, want the single id restriction", name, outer["filter"])+ }+ wrapper, ok := filter[0].(map[string]interface{})if !ok {- t.Fatalf("%s: should is not []interface{}: %T", name, outer["should"])+ t.Fatalf("%s: filter[0] is %T, want map[string]interface{}", name, filter[0])+ }+ inner, ok := wrapper["bool"].(map[string]interface{})+ if !ok {+ t.Fatalf("%s: filter[0] carries no bool: %v", name, wrapper)+ }+ if inner["minimum_should_match"] != 1 {+ t.Fatalf("%s: id bool minimum_should_match = %v, want 1", name, inner["minimum_should_match"])+ }+ should, ok := inner["should"].([]interface{})+ if !ok {+ t.Fatalf("%s: should is not []interface{}: %T", name, inner["should"])}if len(should) != len(wantFields) {t.Fatalf("%s: should length = %d, want %d (raw=%v)", name, len(should), len(wantFields), should)@@ -420,19 +451,13 @@ func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {if !ok {t.Fatalf("%s: should[%d] is %T, want map[string]interface{} (nested array bug) — raw=%v", name, i, el, el)}- if inner, ok := m["term"].(map[string]interface{}); ok {- for f := range inner {- seenFields[f] = true- }- continue+ clause, ok := m[wantOp].(map[string]interface{})+ if !ok {+ t.Fatalf("%s: should[%d] = %v, want %q clause", name, i, m, wantOp)}- if terms, ok := m["terms"].(map[string]interface{}); ok {- for f := range terms {- seenFields[f] = true- }- continue+ for f := range clause {+ seenFields[f] = true}- t.Fatalf("%s: should[%d] missing term/terms: %v", name, i, m)}for _, want := range wantFields {if !seenFields[want] {@@ -443,19 +468,19 @@ func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {check("list_value", map[string]interface{}{"id": []interface{}{"a", "b", "c"},- }, []string{"id", "_id"})+ }, []string{"id", "_id"}, "terms")check("string_list_value", map[string]interface{}{"id": []string{"a", "b", "c"},- }, []string{"id", "_id"})+ }, []string{"id", "_id"}, "terms")check("string_value", map[string]interface{}{"id": "doc-42",- }, []string{"id", "_id"})+ }, []string{"id", "_id"}, "term")check("int_value", map[string]interface{}{"id": 42,- }, []string{"id", "_id"})+ }, []string{"id", "_id"}, "term")// A typed []string must be handled too: callers built from typed helpers// (e.g. list_chunks' ChunkScope) pass []string, and the generic loop below@@ -464,7 +489,69 @@ func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {// window returned 3.8MB instead of ~17KB).check("string_slice_value", map[string]interface{}{"id": []string{"a", "b", "c"},- }, []string{"id", "_id"})+ }, []string{"id", "_id"}, "terms")+}++// TestRankFeatureDoesNotWidenChunkIDScope pins why id must be a filter clause:+// Search appends rank_feature queries to bool.should, so an id restriction+// living in that slot is satisfied by the rank feature alone once+// minimum_should_match=1 is set, and a chunk-id-scoped read returns unrelated+// chunks.+func TestRankFeatureDoesNotWidenChunkIDScope(t *testing.T) {+ var captured map[string]interface{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if err := json.NewDecoder(r.Body).Decode(&captured); err != nil {+ t.Error(err)+ http.Error(w, "bad query", http.StatusBadRequest)+ return+ }+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ w.Header().Set("Content-Type", "application/json")+ if err := json.NewEncoder(w).Encode(map[string]interface{}{+ "hits": map[string]interface{}{+ "total": map[string]interface{}{"value": 0},+ "hits": []interface{}{},+ },+ }); err != nil {+ t.Error(err)+ }+ }))+ defer server.Close()+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatal(err)+ }+ eng := &Engine{client: client}+ if _, err := eng.Search(context.Background(), &types.SearchRequest{+ IndexNames: []string{"ragflow_t1"},+ KbIDs: []string{"kb1"},+ Limit: 10,+ Filter: map[string]interface{}{"id": []string{"chunk-a"}},+ RankFeature: map[string]float64{common.PAGERANK_FLD: 1},+ }); err != nil {+ t.Fatal(err)+ }+ query, ok := captured["query"].(map[string]interface{})+ if !ok {+ t.Fatalf("captured body carries no query: %v", captured)+ }+ boolMap, ok := query["bool"].(map[string]interface{})+ if !ok {+ t.Fatalf("query carries no bool: %v", query)+ }+ if _, exists := boolMap["minimum_should_match"]; exists {+ t.Fatalf("top-level minimum_should_match also binds the rank_feature clauses: %v", boolMap)+ }+ encoded, err := json.Marshal(boolMap["filter"])+ if err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(string(encoded), `"id":["chunk-a"]`) {+ t.Fatalf("id restriction is not a mandatory filter clause: %s", encoded)+ }+ if !strings.Contains(string(encoded), `"minimum_should_match":1`) {+ t.Fatalf("id filter lost the id/_id alternative: %s", encoded)+ }}// paginationGRID mirrors the (page_size, top) grid fromdiff --git a/internal/handler/dify_retrieval_handler.go b/internal/handler/dify_retrieval_handler.goindex 9ee74eb6b3..4e1c8383e6 100644--- a/internal/handler/dify_retrieval_handler.go+++ b/internal/handler/dify_retrieval_handler.go@@ -262,7 +262,9 @@ func (h *DifyRetrievalHandler) Retrieval(c *gin.Context) {docIDs = []string{service.NoMatchDocIDSentinel}}- // Label question for rank features+ // Hand the label result over unconditionally: an empty map says "issue no+ // rank-feature clause", while a nil pointer would let the nlp layer substitute+ // its {pagerank_fea: 10} default and add a clause the reference never sends.kbs := []*entity.Knowledgebase{kb}rankFeature := h.metadataSvc.LabelQuestion(ctx, req.Query, kbs)@@ -283,9 +285,7 @@ func (h *DifyRetrievalHandler) Retrieval(c *gin.Context) {EmbeddingModel: embModel,Language: entity.KnowledgebasesLanguage(kbs),}- if rankFeature != nil {- sr.RankFeature = &rankFeature- }+ sr.RankFeature = &rankFeatureresult, err := h.retrievalSvc.Retrieval(c.Request.Context(), sr)if err != nil {diff --git a/internal/handler/dify_retrieval_handler_test.go b/internal/handler/dify_retrieval_handler_test.goindex e9259729ab..c00b5cc936 100644--- a/internal/handler/dify_retrieval_handler_test.go+++ b/internal/handler/dify_retrieval_handler_test.go@@ -557,3 +557,78 @@ func TestDifyRetrieval_TopKOutOfRange(t *testing.T) {})}}++// captureRetrievalRequest installs a retrieval service that records the request+// it was handed, so a test can assert on the nlp-layer parameters.+func captureRetrievalRequest(h *DifyRetrievalHandler) **nlp.RetrievalRequest {+ captured := new(*nlp.RetrievalRequest)+ h.retrievalSvc = &mockRetrievalService{+ retrievalFn: func(ctx context.Context, req *nlp.RetrievalRequest) (*nlp.RetrievalResult, error) {+ *captured = req+ return &nlp.RetrievalResult{+ Chunks: []map[string]interface{}{+ {"doc_id": "doc1", "docnm_kwd": "Test Doc", "content_with_weight": "test content", "similarity": 0.85},+ },+ }, nil+ },+ }+ return captured+}++// LabelQuestion returning nil (no tag source on the dataset) must still reach the+// retrieval layer as a non-nil pointer to an empty map; a nil pointer makes the+// nlp layer substitute its {pagerank_fea: 10} default.+func TestDifyRetrieval_NoTagSourceSendsEmptyRankFeature(t *testing.T) {+ h, r := setupDifyTest("user1")+ h.metadataSvc = &mockMetadataService{+ labelQuestionFn: func(ctx context.Context, question string, kbs []*entity.Knowledgebase) map[string]float64 {+ return nil+ },+ }+ captured := captureRetrievalRequest(h)++ body := `{"knowledge_id": "kb1", "query": "test question"}`+ w := httptest.NewRecorder()+ req, _ := http.NewRequest("POST", "/api/v1/dify/retrieval", strings.NewReader(body))+ req.Header.Set("Content-Type", "application/json")+ r.ServeHTTP(w, req)++ if w.Code != http.StatusOK {+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())+ }+ if *captured == nil {+ t.Fatal("retrieval service was not called")+ }+ if (*captured).RankFeature == nil {+ t.Fatal("RankFeature = nil, want a pointer to an empty map; nil lets the nlp layer substitute the pagerank_fea default")+ }+ if got := len(*(*captured).RankFeature); got != 0 {+ t.Fatalf("RankFeature has %d entries (%v), want empty", got, *(*captured).RankFeature)+ }+}++func TestDifyRetrieval_TagSourceForwardsRankFeatures(t *testing.T) {+ h, r := setupDifyTest("user1")+ h.metadataSvc = &mockMetadataService{+ labelQuestionFn: func(ctx context.Context, question string, kbs []*entity.Knowledgebase) map[string]float64 {+ return map[string]float64{"tag_1": 0.8}+ },+ }+ captured := captureRetrievalRequest(h)++ body := `{"knowledge_id": "kb1", "query": "test question"}`+ w := httptest.NewRecorder()+ req, _ := http.NewRequest("POST", "/api/v1/dify/retrieval", strings.NewReader(body))+ req.Header.Set("Content-Type", "application/json")+ r.ServeHTTP(w, req)++ if w.Code != http.StatusOK {+ t.Fatalf("expected 200, got %d: %s", w.Code, w.Body.String())+ }+ if *captured == nil || (*captured).RankFeature == nil {+ t.Fatal("RankFeature was not forwarded to the retrieval layer")+ }+ if got := (*(*captured).RankFeature)["tag_1"]; got != 0.8 {+ t.Fatalf("RankFeature[tag_1] = %v, want 0.8", got)+ }+}diff --git a/internal/harness/core/cancel_full_test.go b/internal/harness/core/cancel_full_test.goindex 2d89fd8d40..582c9f49da 100644--- a/internal/harness/core/cancel_full_test.go+++ b/internal/harness/core/cancel_full_test.go@@ -1238,6 +1238,21 @@ func TestAgentCancelFunc_MultiCall_JoinSafePointModes(t *testing.T) {}}+// waitUntil polls cond until it holds or limit elapses. The timeout escalation+// runs on its own goroutine, so a fixed sleep races with scheduler starvation+// when the whole test suite runs in parallel.+func waitUntil(t *testing.T, limit time.Duration, cond func() bool) bool {+ t.Helper()+ deadline := time.Now().Add(limit)+ for time.Now().Before(deadline) {+ if cond() {+ return true+ }+ time.Sleep(5 * time.Millisecond)+ }+ return cond()+}+func TestAgentCancelFunc_MultiCall_TimeoutEscalationReturnsErrCancelTimeout(t *testing.T) {cc := newCancelContext()cf := cc.buildCancelFunc()@@ -1245,9 +1260,8 @@ func TestAgentCancelFunc_MultiCall_TimeoutEscalationReturnsErrCancelTimeout(t *tif !ok {t.Fatal("first should contribute")}- time.Sleep(50 * time.Millisecond)- if !cc.isImmediate() {- t.Error("should escalate to immediate")+ if !waitUntil(t, 5*time.Second, cc.isImmediate) {+ t.Fatal("should escalate to immediate")}if atomic.LoadInt32(&cc.timeoutEscalated) != 1 {t.Error("expected timeout escalated")@@ -1261,13 +1275,28 @@ func TestAgentCancelFunc_MultiCall_TimeoutDeadlineJoinUsesAbsoluteTime(t *testinif !ok1 {t.Fatal("first should contribute")}+ firstDeadline := atomic.LoadInt64(&cc.deadlineUnixNano)+ if firstDeadline == 0 {+ t.Fatal("first call must arm a deadline")+ }++ joinAt := time.Now()_, ok2 := cf(WithCancelMode(CancelAfterChatModel), WithCancelTimeout(20*time.Millisecond))if !ok2 {t.Fatal("second should contribute")}- time.Sleep(50 * time.Millisecond)- if !cc.isImmediate() {- t.Error("should escalate to immediate after short timeout")+ // The join keeps the earliest absolute deadline: a 20ms registration made+ // right after a 200ms one must replace it, which only holds when deadlines+ // are compared as absolute times instead of durations.+ joined := atomic.LoadInt64(&cc.deadlineUnixNano)+ if joined >= firstDeadline {+ t.Fatalf("deadline = %d, first = %d; the 20ms registration must shorten the 200ms one", joined, firstDeadline)+ }+ if limit := joinAt.Add(50 * time.Millisecond).UnixNano(); joined > limit {+ t.Fatalf("deadline = %d, want roughly join+20ms (<= %d)", joined, limit)+ }+ if !waitUntil(t, 5*time.Second, cc.isImmediate) {+ t.Fatal("should escalate to immediate after short timeout")}}diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.goindex b6eaafd08e..4dfd2d5055 100644--- a/internal/ingestion/task/indexdoc/process.go+++ b/internal/ingestion/task/indexdoc/process.go@@ -105,6 +105,21 @@ func ProcessChunksForPipeline(return metadata, nil}+// StampDatasetPagerank writes the dataset's pagerank onto every chunk a run is+// about to index. Python's index-time doc template carries it+// (`if task["pagerank"]: doc[PAGERANK_FLD] = int(...)`), which is what makes a+// pagerank configured before parsing apply to the chunks it produces. Without+// it only the chunks that already existed when the setting was saved carry the+// value, since the dataset-level update rewrites rows rather than future ones.+func StampDatasetPagerank(chunks []map[string]any, pagerank int64) {+ if pagerank == 0 {+ return+ }+ for _, chunk := range chunks {+ chunk[common.PAGERANK_FLD] = pagerank+ }+}+// requireStringText enforces the pre-index wire contract: every chunk must// carry a string "text" field before chunk-id generation or persistence mapping.func requireStringText(ck map[string]any) (string, error) {diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex adb07e1c27..8059cd138e 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -12,6 +12,27 @@ import ("ragflow/internal/parser/parser")+// TestStampDatasetPagerank pins the index-time stamp: a configured dataset+// pagerank lands on every chunk the run indexes, and zero leaves chunks+// untouched (Python only stamps a truthy pagerank).+func TestStampDatasetPagerank(t *testing.T) {+ chunks := []map[string]any{{"id": "a"}, {"id": "b"}}++ StampDatasetPagerank(chunks, 0)+ for _, chunk := range chunks {+ if _, ok := chunk[common.PAGERANK_FLD]; ok {+ t.Fatalf("zero pagerank stamped a field: %#v", chunk)+ }+ }++ StampDatasetPagerank(chunks, 7)+ for _, chunk := range chunks {+ if got := chunk[common.PAGERANK_FLD]; got != int64(7) {+ t.Fatalf("pagerank_fea = %#v, want 7", got)+ }+ }+}+func TestProcessChunksForPipeline_PreservesTypedMetadata(t *testing.T) {want := map[string]any{"year": float64(2025), "verified": false, "scalar": map[string]any{"x": 1},diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 84d02d0d7a..727de8f315 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -338,6 +338,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput mapreturn nil, err}indexChunks := append(chunks, parentChunks...)+ indexdoc.StampDatasetPagerank(indexChunks, s.taskCtx.KB.Pagerank)if err := s.indexWriter.Write(ctx, indexChunks); err != nil {if cleanupErr := s.compensateFailedIndexWrite(ctx, indexChunks); cleanupErr != nil {return nil, fmt.Errorf("write chunks: %w; compensate partial index write: %v", err, cleanupErr)diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex 3ee6d51d8a..b6025c5b93 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -1479,6 +1479,12 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequesif tagFeas != nil {chunkData["tag_feas"] = tagFeas}+ if kb.Pagerank != 0 {+ // Python's chunk_api.create stamps the dataset's pagerank on a manually+ // added chunk (`if kb.pagerank: d["pagerank_fea"] = kb.pagerank`); a+ // chunk that skipped it would rank at 0 until the setting is re-saved.+ chunkData[common.PAGERANK_FLD] = kb.Pagerank+ }if req.ImageBase64 != nil {var imageBinary []bytediff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.goindex cbe742fbda..75d62fbe22 100644--- a/internal/service/chunk/chunk_test.go+++ b/internal/service/chunk/chunk_test.go@@ -1159,7 +1159,7 @@ func TestAddChunkSuccess(t *testing.T) {return permission.ErrPermissionDenied},getKnowledgebaseByIDFunc: func(id string) (*entity.Knowledgebase, error) {- return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1", Language: &language}, nil+ return &entity.Knowledgebase{ID: id, TenantID: userID, EmbdID: "embed-1", Language: &language, Pagerank: 7}, nil},getEmbeddingModelFunc: func(string, string) (*models.EmbeddingModel, error) {driver := &stubEmbeddingDriver{@@ -1210,6 +1210,12 @@ func TestAddChunkSuccess(t *testing.T) {if resp.Chunk["content"] != "chunk body" {t.Fatalf("content = %v, want chunk body", resp.Chunk["content"])}+ if len(engine.insertedChunks) != 1 {+ t.Fatalf("inserted chunks = %d, want 1", len(engine.insertedChunks))+ }+ if got := engine.insertedChunks[0][common.PAGERANK_FLD]; got != int64(7) {+ t.Fatalf("pagerank_fea = %v, want the dataset's 7", got)+ }if resp.Chunk["document"] != "doc-1.txt" {t.Fatalf("document = %v, want doc-1.txt", resp.Chunk["document"])}diff --git a/internal/service/nlp/reranker.go b/internal/service/nlp/reranker.goindex 76b32c574a..79217a83eb 100644--- a/internal/service/nlp/reranker.go+++ b/internal/service/nlp/reranker.go@@ -1055,6 +1055,12 @@ func toFloat64(v interface{}) (float64, bool) {return float64(val), truecase int32:return float64(val), true+ case string:+ // The engine's field projection stringifies non-string values+ // (pagerank_fea arrives as "7"), and Python's rerank coerces the same+ // string through np.array(..., dtype=float).+ f, err := strconv.ParseFloat(strings.TrimSpace(val), 64)+ return f, err == nildefault:return 0, false}diff --git a/internal/service/nlp/reranker_rank_feature_test.go b/internal/service/nlp/reranker_rank_feature_test.goindex 2b85150de1..37ccd3dd66 100644--- a/internal/service/nlp/reranker_rank_feature_test.go+++ b/internal/service/nlp/reranker_rank_feature_test.go@@ -85,6 +85,41 @@ func TestApplyRankFeatureScores_PagerankOnly(t *testing.T) {}}+// TestApplyRankFeatureScores_StringifiedPagerank pins the engine field+// projection: GetFields stringifies non-string values, so a chunk's pagerank+// reaches the reranker as "7" and must still count (Python coerces it through+// np.array(..., dtype=float)).+func TestApplyRankFeatureScores_StringifiedPagerank(t *testing.T) {+ chunks := []map[string]interface{}{+ {common.PAGERANK_FLD: "7"},+ {common.PAGERANK_FLD: 3},+ }+ got := applyRankFeatureScores(chunks, []float64{1, 1}, nil)+ want := []float64{8, 4}+ if !floatsClose(got, want, 1e-9) {+ t.Fatalf("got %v, want %v", got, want)+ }+}++// TestApplyRankFeatureScoresForIDs_ProjectionShapes mirrors the ES field map as+// the reranker actually receives it: numbers stringified and tag_feas rendered+// as JSON.+func TestApplyRankFeatureScoresForIDs_ProjectionShapes(t *testing.T) {+ field := map[string]map[string]interface{}{+ "a": {common.TAG_FLD: `{"tag_a":0.5}`, common.PAGERANK_FLD: "5"},+ "b": {common.PAGERANK_FLD: "0"},+ }+ rankFeature := map[string]float64{"tag_a": 1}+ got := applyRankFeatureScoresForIDs([]string{"a", "b"}, field, []float64{0.2, 0.2}, rankFeature)+ want := []float64{+ wantRankFeature(0.2, map[string]float64{"tag_a": 0.5}, 5, rankFeature),+ 0.2,+ }+ if !floatsClose(got, want, 1e-9) {+ t.Fatalf("got %v, want %v", got, want)+ }+}+func TestApplyRankFeatureScores_PagerankOnlyQueryNoDivZero(t *testing.T) {chunks := []map[string]interface{}{{common.PAGERANK_FLD: 10}}sim := []float64{0.5}diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex 29e3555519..b5260c680e 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -556,7 +556,13 @@ func (s *RetrievalService) scoreSearchResult(ctx context.Context, req *Retrievalfor i, id := range searchResult.IDs {sim[i] = scores[id]}- return sim, make([]float64, len(sim)), sim, nil+ // A dense-only request has no scored bool query, so the engine never+ // applies its rank_feature clauses; the rank features have to be added+ // here to reach the similarity at all.+ // The boost mutates sim, so vector_similarity is sampled first.+ rawSim := slices.Clone(sim)+ sim = applyRankFeatureScoresForIDs(searchResult.IDs, searchResult.Field, sim, *req.RankFeature)+ return sim, make([]float64, len(sim)), rawSim, nil}knnResult, err := s.docEngine.KNNScores(ctx, searchResult.Chunks, searchResult.QueryVector, len(searchResult.IDs))@@ -728,6 +734,14 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque// child fragments to their parent, row_id() the table row identity."content_with_weight", "doc_type_kwd", "mom_id", "row_id()","compile_kwd", "scope_kwd",+ // Generated questions: question_tks feeds the reranker's term input (×6),+ // question_kwd is what the dataset search API reports as "questions".+ "question_tks", "question_kwd",+ // Rank-feature inputs. The reranker reads them off searchResult.Field,+ // which is the engine's projection of exactly this list, so leaving them+ // out scored every chunk at pagerank 0 and dropped the tag-feature+ // weight entirely. Python's default src carries both.+ common.PAGERANK_FLD, common.TAG_FLD,"_score",}@@ -806,7 +820,8 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque// Build source with vector column for ESsearchSrc := make([]string, len(searchRequest.SelectFields))copy(searchSrc, searchRequest.SelectFields)- if s.docEngine.GetType() == string(engine.EngineElasticsearch) || engine.IsOceanBaseFamily(s.docEngine.GetType()) || engine.IsVastbase(s.docEngine.GetType()) {+ driverType := s.docEngine.GetType()+ if driverType == string(engine.EngineElasticsearch) || engine.IsOceanBaseFamily(driverType) || engine.IsVastbase(driverType) {searchSrc = append(searchSrc, matchDense.VectorColumnName)}diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 4f512eb2c5..b8e04a27d1 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -194,6 +194,164 @@ func TestRetrievalPreservesCompilationKindWhenPresent(t *testing.T) {}}+// TestRetrievalProjectsRankFeatureFields pins the source projection. The+// reranker reads pagerank_fea and tag_feas off Field, and Field is the engine's+// projection of exactly SelectFields, so both must be requested.+func TestRetrievalProjectsRankFeatureFields(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{+ {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},+ }}+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ threshold := 0.0+ vectorWeight := 1.0+ aggs := false+ if _, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 1,+ PageSize: 1,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ }); err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ if len(docEngine.selectedFields) == 0 {+ t.Fatal("no search was issued")+ }+ for _, fields := range docEngine.selectedFields {+ if !slices.Contains(fields, common.PAGERANK_FLD) || !slices.Contains(fields, common.TAG_FLD) {+ t.Fatalf("search fields = %v, want %s and %s", fields, common.PAGERANK_FLD, common.TAG_FLD)+ }+ }+}++// TestRetrievalProjectsQuestionTokenFields pins the same projection contract for+// the generated-question fields. The reranker folds question_tks into its term+// input with a ×6 weight and the dataset search API reports question_kwd as+// "questions", but both are read off Field — the engine's projection of exactly+// SelectFields — so omitting them scores those tokens as absent.+func TestRetrievalProjectsQuestionTokenFields(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{+ {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "question_tks": "beta", "_score": 0.9},+ }}+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ threshold := 0.0+ vectorWeight := 1.0+ aggs := false+ if _, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 1,+ PageSize: 1,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ }); err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ if len(docEngine.selectedFields) == 0 {+ t.Fatal("no search was issued")+ }+ for _, fields := range docEngine.selectedFields {+ if !slices.Contains(fields, "question_tks") || !slices.Contains(fields, "question_kwd") {+ t.Fatalf("search fields = %v, want question_tks and question_kwd", fields)+ }+ }+}++// TestRetrievalPagerankBoostsRanking drives the whole path: identical chunks,+// only pagerank_fea differs, so the boosted chunk must win and its similarity+// must carry the pagerank on top of the hybrid score.+func TestRetrievalPagerankBoostsRanking(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{+ {"id": "plain", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5},+ {"id": "boosted", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(4)},+ }}+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ threshold := 0.0+ vectorWeight := 1.0+ aggs := false+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 1,+ PageSize: 2,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ })+ if err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ if len(result.Chunks) != 2 {+ t.Fatalf("chunks = %d, want 2", len(result.Chunks))+ }+ if got := result.Chunks[0]["chunk_id"]; got != "boosted" {+ t.Fatalf("first chunk = %v, want boosted (pagerank must outrank the tie)", got)+ }+ boosted, _ := result.Chunks[0]["similarity"].(float64)+ plain, _ := result.Chunks[1]["similarity"].(float64)+ if boosted-plain < 4-1e-9 {+ t.Fatalf("similarity delta = %v, want the 4-point pagerank boost", boosted-plain)+ }+}++// TestRetrievalVectorOnlyKeepsPagerank covers the dense-only path: the engine+// issues no scored bool query there, so the engine can never apply its+// rank_feature clause and the similarity must pick pagerank up locally.+// vector_similarity keeps the engine's raw kNN score (asserted below).+func TestRetrievalVectorOnlyKeepsPagerank(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{+ {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(3)},+ }}+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ threshold := 0.0+ vectorWeight := 1.0+ aggs := false+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 1,+ PageSize: 1,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ VectorOnly: true,+ })+ if err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ if len(result.Chunks) != 1 {+ t.Fatalf("chunks = %d, want 1", len(result.Chunks))+ }+ if got, _ := result.Chunks[0]["similarity"].(float64); got != 3.5 {+ t.Fatalf("similarity = %v, want engine score 0.5 + pagerank 3", got)+ }+ if got, _ := result.Chunks[0]["vector_similarity"].(float64); got != 0.5 {+ t.Fatalf("vector_similarity = %v, want the raw kNN score 0.5 (rank features belong to similarity only)", got)+ }+ if got, _ := result.Chunks[0]["term_similarity"].(float64); got != 0 {+ t.Fatalf("term_similarity = %v, want 0 for a dense-only request", got)+ }+}+func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()