RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: expose compile_kwd in dataset search results (#20859)
Liang Hongwei5556dfdd378f
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.goindex 14e16883c5..0f232ca005 100644--- a/internal/service/dataset/search.go+++ b/internal/service/dataset/search.go@@ -329,7 +329,22 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search}filteredChunks = nlp.RetrievalByChildren(filteredChunks, tenantIDs, d.docEngine, ctx)+ formatDatasetSearchChunks(filteredChunks, req.IncludeCompiledChunks)+ common.Info("SearchDatasets completed", zap.String("userID", userID), zap.Any("kbID", datasetIDs), zap.String("question", question), zap.Int64("chunkCount", int64(len(filteredChunks))))++ pyChunks := common.ConvertFloatsToPyFormat(filteredChunks).([]map[string]interface{})++ return &service.SearchDatasetsResponse{+ Chunks: pyChunks,+ DocAggs: retrievalResult.DocAggs,+ Labels: &labels,+ Total: retrievalResult.Total,+ }, nil+}++// formatDatasetSearchChunks applies API field names and compilation visibility after parent promotion.+func formatDatasetSearchChunks(filteredChunks []map[string]interface{}, includeCompiledChunks *bool) {keyMapping := map[string]string{"chunk_id": "id","content_with_weight": "content",@@ -340,6 +355,9 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search"kb_id": "dataset_id",}for i := range filteredChunks {+ if includeCompiledChunks != nil && !*includeCompiledChunks {+ delete(filteredChunks[i], "compile_kwd")+ }delete(filteredChunks[i], "vector")for oldKey, newKey := range keyMapping {if value, ok := filteredChunks[i][oldKey]; ok {@@ -348,17 +366,6 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search}}}-- common.Info("SearchDatasets completed", zap.String("userID", userID), zap.Any("kbID", datasetIDs), zap.String("question", question), zap.Int64("chunkCount", int64(len(filteredChunks))))-- pyChunks := common.ConvertFloatsToPyFormat(filteredChunks).([]map[string]interface{})-- return &service.SearchDatasetsResponse{- Chunks: pyChunks,- DocAggs: retrievalResult.DocAggs,- Labels: &labels,- Total: retrievalResult.Total,- }, nil}func knowledgeCompilationFilter(include *bool, scope string) map[string]interface{} {diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.goindex e17fea8e9f..fa552ac432 100644--- a/internal/service/dataset/search_test.go+++ b/internal/service/dataset/search_test.go@@ -1,6 +1,7 @@package datasetimport (+ "context""reflect""slices""testing"@@ -191,3 +192,86 @@ func TestSelectMetadataFilteredDocIDs(t *testing.T) {})}}++func TestFormatDatasetSearchChunksCompilationVisibility(t *testing.T) {+ enabled, disabled := true, false+ for _, tt := range []struct {+ name string+ include *bool+ present bool+ }{+ {name: "disabled", include: &disabled},+ {name: "enabled", include: &enabled, present: true},+ {name: "unspecified", present: true},+ } {+ t.Run(tt.name, func(t *testing.T) {+ chunks := []map[string]interface{}{+ {"chunk_id": "compiled-1", "content_with_weight": "compiled content", "compile_kwd": "page_index", "vector": []float64{1}},+ {"chunk_id": "ordinary-1", "content_with_weight": "ordinary content"},+ }+ formatDatasetSearchChunks(chunks, tt.include)+ if len(chunks) != 2 || chunks[0]["id"] != "compiled-1" || chunks[0]["content"] != "compiled content" {+ t.Fatalf("API fields changed: %#v", chunks)+ }+ kind, present := chunks[0]["compile_kwd"]+ if present != tt.present || (present && kind != "page_index") {+ t.Fatalf("compile_kwd = %v, present = %v; want present = %v", kind, present, tt.present)+ }+ if _, present := chunks[1]["compile_kwd"]; present {+ t.Fatalf("ordinary chunk has compilation marker: %#v", chunks[1])+ }+ if _, present := chunks[0]["vector"]; present {+ t.Fatalf("API chunk still has vector: %#v", chunks[0])+ }+ })+ }+}++func TestDisabledCompilationStillPromotesOrdinaryParent(t *testing.T) {+ disabled := false+ eng := &compilationSearchEngine{rows: []map[string]interface{}{+ {"id": "compiled", "compile_kwd": "page_index", "available_int": 1},+ {"id": "child-1", "mom_id": "parent-1", "doc_id": "doc-1", "kb_id": "kb-1", "available_int": 1},+ {"id": "parent-1", "doc_id": "doc-1", "kb_id": "kb-1", "content_with_weight": "parent content", "available_int": 0},+ }}+ filter := knowledgeCompilationFilter(&disabled, "all")+ filter["available_int"] = 1+ hits, err := eng.Search(t.Context(), &types.SearchRequest{Filter: filter})+ if err != nil {+ t.Fatal(err)+ }+ if len(hits.Chunks) != 1 || hits.Chunks[0]["id"] != "child-1" {+ t.Fatalf("initial hits = %#v, want only the ordinary child", hits.Chunks)+ }+ hits.Chunks[0]["chunk_id"] = hits.Chunks[0]["id"]+ chunks := nlp.RetrievalByChildren(hits.Chunks, []string{"tenant-1"}, eng, t.Context())+ formatDatasetSearchChunks(chunks, &disabled)+ if len(chunks) != 1 || chunks[0]["id"] != "parent-1" || chunks[0]["content"] != "parent content" {+ t.Fatalf("parent promotion changed: %#v", chunks)+ }+ if _, present := chunks[0]["compile_kwd"]; present {+ t.Fatalf("disabled response has compilation marker: %#v", chunks[0])+ }+}++type compilationSearchEngine struct {+ fakeChatDocEngine+ rows []map[string]interface{}+}++func (e *compilationSearchEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ var chunks []map[string]interface{}+ for _, row := range e.rows {+ if !matchesKnowledgeCompilationFilter(req.Filter, row) {+ continue+ }+ chunk := make(map[string]interface{}, len(row))+ for field, value := range row {+ if len(req.SelectFields) == 0 || field == "id" || slices.Contains(req.SelectFields, field) {+ chunk[field] = value+ }+ }+ chunks = append(chunks, chunk)+ }+ return &types.SearchResult{Chunks: chunks, Total: int64(len(chunks))}, nil+}diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex 9170eb9189..ea8911fe82 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -302,6 +302,9 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)if v, ok := chunk["kb_id"]; ok {resultChunk["kb_id"] = v}+ if kind := firstStringValue(chunk["compile_kwd"]); kind != "" {+ resultChunk["compile_kwd"] = kind+ }if v, ok := chunk["important_kwd"]; ok {resultChunk["important_kwd"] = v}diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex e13e8096b2..f0eacd464a 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -76,6 +76,47 @@ func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {}}+func TestRetrievalPreservesCompilationKindWhenPresent(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{+ {"id": "compiled", "content_ltks": "alpha", "content_with_weight": "alpha", "compile_kwd": "page_index", "_score": 0.9},+ {"id": "ordinary", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},+ }}+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ threshold := 0.0+ vectorWeight := 1.0+ aggs := false+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 1,+ PageSize: 2,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ })+ if err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ for _, fields := range docEngine.selectedFields {+ if !slices.Contains(fields, "compile_kwd") {+ t.Fatalf("search fields = %v, want compile_kwd", fields)+ }+ }+ if len(result.Chunks) != 2 {+ t.Fatalf("chunks = %#v, want compiled and ordinary chunks", result.Chunks)+ }+ if got := result.Chunks[0]["compile_kwd"]; got != "page_index" {+ t.Fatalf("compiled chunk compile_kwd = %v, want page_index", got)+ }+ if _, ok := result.Chunks[1]["compile_kwd"]; ok {+ t.Fatalf("ordinary chunk unexpectedly has compile_kwd: %#v", result.Chunks[1])+ }+}+func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()@@ -202,6 +243,7 @@ type retrievalCountEngine struct {rows []map[string]interface{}searchLimits []intsearchFilters []map[string]interface{}+ selectedFields [][]stringhighlights map[string]stringhighlightCalls int}@@ -209,6 +251,7 @@ type retrievalCountEngine struct {func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {e.searchLimits = append(e.searchLimits, req.Limit)e.searchFilters = append(e.searchFilters, req.Filter)+ e.selectedFields = append(e.selectedFields, slices.Clone(req.SelectFields))offset := req.Offsetif offset > len(e.rows) {offset = len(e.rows)@@ -217,7 +260,21 @@ func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchRequesif req.Limit <= 0 || end > len(e.rows) {end = len(e.rows)}- return &types.SearchResult{Chunks: e.rows[offset:end], Total: int64(len(e.rows))}, nil+ chunks := make([]map[string]interface{}, 0, end-offset)+ for _, row := range e.rows[offset:end] {+ if len(req.SelectFields) == 0 {+ chunks = append(chunks, maps.Clone(row))+ continue+ }+ projected := map[string]interface{}{"id": row["id"]}+ for _, field := range req.SelectFields {+ if value, ok := row[field]; ok {+ projected[field] = value+ }+ }+ chunks = append(chunks, projected)+ }+ return &types.SearchResult{Chunks: chunks, Total: int64(len(e.rows))}, nil}func (e *retrievalCountEngine) GetChunkIDs(chunks []map[string]interface{}) []string {@@ -230,11 +287,17 @@ func (e *retrievalCountEngine) GetChunkIDs(chunks []map[string]interface{}) []streturn ids}-func (e *retrievalCountEngine) GetFields(chunks []map[string]interface{}, _ []string) map[string]map[string]interface{} {+func (e *retrievalCountEngine) GetFields(chunks []map[string]interface{}, selected []string) map[string]map[string]interface{} {fields := make(map[string]map[string]interface{}, len(chunks))for _, chunk := range chunks {if id, ok := chunk["id"].(string); ok {- fields[id] = chunk+ projected := make(map[string]interface{}, len(selected))+ for _, field := range selected {+ if value, ok := chunk[field]; ok {+ projected[field] = value+ }+ }+ fields[id] = projected}}return fields