Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: expose compile_kwd in dataset search results (#20859)

Liang Hongwei5556dfdd378f
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go
index 14e16883c5..0f232ca005 100644
--- a/internal/service/dataset/search.go
+++ b/internal/service/dataset/search.go
@@ -329,7 +329,22 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
}
filteredChunks = nlp.RetrievalByChildren(filteredChunks, tenantIDs, d.docEngine, ctx)
+ formatDatasetSearchChunks(filteredChunks, req.IncludeCompiledChunks)
+ common.Info("SearchDatasets completed", zap.String("userID", userID), zap.Any("kbID", datasetIDs), zap.String("question", question), zap.Int64("chunkCount", int64(len(filteredChunks))))
+
+ pyChunks := common.ConvertFloatsToPyFormat(filteredChunks).([]map[string]interface{})
+
+ return &service.SearchDatasetsResponse{
+ Chunks: pyChunks,
+ DocAggs: retrievalResult.DocAggs,
+ Labels: &labels,
+ Total: retrievalResult.Total,
+ }, nil
+}
+
+// formatDatasetSearchChunks applies API field names and compilation visibility after parent promotion.
+func formatDatasetSearchChunks(filteredChunks []map[string]interface{}, includeCompiledChunks *bool) {
keyMapping := map[string]string{
"chunk_id": "id",
"content_with_weight": "content",
@@ -340,6 +355,9 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
"kb_id": "dataset_id",
}
for i := range filteredChunks {
+ if includeCompiledChunks != nil && !*includeCompiledChunks {
+ delete(filteredChunks[i], "compile_kwd")
+ }
delete(filteredChunks[i], "vector")
for oldKey, newKey := range keyMapping {
if value, ok := filteredChunks[i][oldKey]; ok {
@@ -348,17 +366,6 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
}
}
}
-
- common.Info("SearchDatasets completed", zap.String("userID", userID), zap.Any("kbID", datasetIDs), zap.String("question", question), zap.Int64("chunkCount", int64(len(filteredChunks))))
-
- pyChunks := common.ConvertFloatsToPyFormat(filteredChunks).([]map[string]interface{})
-
- return &service.SearchDatasetsResponse{
- Chunks: pyChunks,
- DocAggs: retrievalResult.DocAggs,
- Labels: &labels,
- Total: retrievalResult.Total,
- }, nil
}
func knowledgeCompilationFilter(include *bool, scope string) map[string]interface{} {
diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.go
index e17fea8e9f..fa552ac432 100644
--- a/internal/service/dataset/search_test.go
+++ b/internal/service/dataset/search_test.go
@@ -1,6 +1,7 @@
package dataset
import (
+ "context"
"reflect"
"slices"
"testing"
@@ -191,3 +192,86 @@ func TestSelectMetadataFilteredDocIDs(t *testing.T) {
})
}
}
+
+func TestFormatDatasetSearchChunksCompilationVisibility(t *testing.T) {
+ enabled, disabled := true, false
+ for _, tt := range []struct {
+ name string
+ include *bool
+ present bool
+ }{
+ {name: "disabled", include: &disabled},
+ {name: "enabled", include: &enabled, present: true},
+ {name: "unspecified", present: true},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ chunks := []map[string]interface{}{
+ {"chunk_id": "compiled-1", "content_with_weight": "compiled content", "compile_kwd": "page_index", "vector": []float64{1}},
+ {"chunk_id": "ordinary-1", "content_with_weight": "ordinary content"},
+ }
+ formatDatasetSearchChunks(chunks, tt.include)
+ if len(chunks) != 2 || chunks[0]["id"] != "compiled-1" || chunks[0]["content"] != "compiled content" {
+ t.Fatalf("API fields changed: %#v", chunks)
+ }
+ kind, present := chunks[0]["compile_kwd"]
+ if present != tt.present || (present && kind != "page_index") {
+ t.Fatalf("compile_kwd = %v, present = %v; want present = %v", kind, present, tt.present)
+ }
+ if _, present := chunks[1]["compile_kwd"]; present {
+ t.Fatalf("ordinary chunk has compilation marker: %#v", chunks[1])
+ }
+ if _, present := chunks[0]["vector"]; present {
+ t.Fatalf("API chunk still has vector: %#v", chunks[0])
+ }
+ })
+ }
+}
+
+func TestDisabledCompilationStillPromotesOrdinaryParent(t *testing.T) {
+ disabled := false
+ eng := &compilationSearchEngine{rows: []map[string]interface{}{
+ {"id": "compiled", "compile_kwd": "page_index", "available_int": 1},
+ {"id": "child-1", "mom_id": "parent-1", "doc_id": "doc-1", "kb_id": "kb-1", "available_int": 1},
+ {"id": "parent-1", "doc_id": "doc-1", "kb_id": "kb-1", "content_with_weight": "parent content", "available_int": 0},
+ }}
+ filter := knowledgeCompilationFilter(&disabled, "all")
+ filter["available_int"] = 1
+ hits, err := eng.Search(t.Context(), &types.SearchRequest{Filter: filter})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(hits.Chunks) != 1 || hits.Chunks[0]["id"] != "child-1" {
+ t.Fatalf("initial hits = %#v, want only the ordinary child", hits.Chunks)
+ }
+ hits.Chunks[0]["chunk_id"] = hits.Chunks[0]["id"]
+ chunks := nlp.RetrievalByChildren(hits.Chunks, []string{"tenant-1"}, eng, t.Context())
+ formatDatasetSearchChunks(chunks, &disabled)
+ if len(chunks) != 1 || chunks[0]["id"] != "parent-1" || chunks[0]["content"] != "parent content" {
+ t.Fatalf("parent promotion changed: %#v", chunks)
+ }
+ if _, present := chunks[0]["compile_kwd"]; present {
+ t.Fatalf("disabled response has compilation marker: %#v", chunks[0])
+ }
+}
+
+type compilationSearchEngine struct {
+ fakeChatDocEngine
+ rows []map[string]interface{}
+}
+
+func (e *compilationSearchEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ var chunks []map[string]interface{}
+ for _, row := range e.rows {
+ if !matchesKnowledgeCompilationFilter(req.Filter, row) {
+ continue
+ }
+ chunk := make(map[string]interface{}, len(row))
+ for field, value := range row {
+ if len(req.SelectFields) == 0 || field == "id" || slices.Contains(req.SelectFields, field) {
+ chunk[field] = value
+ }
+ }
+ chunks = append(chunks, chunk)
+ }
+ return &types.SearchResult{Chunks: chunks, Total: int64(len(chunks))}, nil
+}
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index 9170eb9189..ea8911fe82 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -302,6 +302,9 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)
if v, ok := chunk["kb_id"]; ok {
resultChunk["kb_id"] = v
}
+ if kind := firstStringValue(chunk["compile_kwd"]); kind != "" {
+ resultChunk["compile_kwd"] = kind
+ }
if v, ok := chunk["important_kwd"]; ok {
resultChunk["important_kwd"] = v
}
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index e13e8096b2..f0eacd464a 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -76,6 +76,47 @@ func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {
}
}
+func TestRetrievalPreservesCompilationKindWhenPresent(t *testing.T) {
+ oldQueryBuilder := globalQueryBuilder
+ globalQueryBuilder = NewQueryBuilder()
+ defer func() { globalQueryBuilder = oldQueryBuilder }()
+
+ docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
+ {"id": "compiled", "content_ltks": "alpha", "content_with_weight": "alpha", "compile_kwd": "page_index", "_score": 0.9},
+ {"id": "ordinary", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},
+ }}
+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
+ threshold := 0.0
+ vectorWeight := 1.0
+ aggs := false
+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{
+ Question: "alpha",
+ TenantIDs: []string{"tenant-1"},
+ Page: 1,
+ PageSize: 2,
+ SimilarityThreshold: &threshold,
+ VectorSimilarityWeight: &vectorWeight,
+ Aggs: &aggs,
+ })
+ if err != nil {
+ t.Fatalf("Retrieval failed: %v", err)
+ }
+ for _, fields := range docEngine.selectedFields {
+ if !slices.Contains(fields, "compile_kwd") {
+ t.Fatalf("search fields = %v, want compile_kwd", fields)
+ }
+ }
+ if len(result.Chunks) != 2 {
+ t.Fatalf("chunks = %#v, want compiled and ordinary chunks", result.Chunks)
+ }
+ if got := result.Chunks[0]["compile_kwd"]; got != "page_index" {
+ t.Fatalf("compiled chunk compile_kwd = %v, want page_index", got)
+ }
+ if _, ok := result.Chunks[1]["compile_kwd"]; ok {
+ t.Fatalf("ordinary chunk unexpectedly has compile_kwd: %#v", result.Chunks[1])
+ }
+}
+
func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
@@ -202,6 +243,7 @@ type retrievalCountEngine struct {
rows []map[string]interface{}
searchLimits []int
searchFilters []map[string]interface{}
+ selectedFields [][]string
highlights map[string]string
highlightCalls int
}
@@ -209,6 +251,7 @@ type retrievalCountEngine struct {
func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
e.searchLimits = append(e.searchLimits, req.Limit)
e.searchFilters = append(e.searchFilters, req.Filter)
+ e.selectedFields = append(e.selectedFields, slices.Clone(req.SelectFields))
offset := req.Offset
if offset > len(e.rows) {
offset = len(e.rows)
@@ -217,7 +260,21 @@ func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchReques
if req.Limit <= 0 || end > len(e.rows) {
end = len(e.rows)
}
- return &types.SearchResult{Chunks: e.rows[offset:end], Total: int64(len(e.rows))}, nil
+ chunks := make([]map[string]interface{}, 0, end-offset)
+ for _, row := range e.rows[offset:end] {
+ if len(req.SelectFields) == 0 {
+ chunks = append(chunks, maps.Clone(row))
+ continue
+ }
+ projected := map[string]interface{}{"id": row["id"]}
+ for _, field := range req.SelectFields {
+ if value, ok := row[field]; ok {
+ projected[field] = value
+ }
+ }
+ chunks = append(chunks, projected)
+ }
+ return &types.SearchResult{Chunks: chunks, Total: int64(len(e.rows))}, nil
}
func (e *retrievalCountEngine) GetChunkIDs(chunks []map[string]interface{}) []string {
@@ -230,11 +287,17 @@ func (e *retrievalCountEngine) GetChunkIDs(chunks []map[string]interface{}) []st
return ids
}
-func (e *retrievalCountEngine) GetFields(chunks []map[string]interface{}, _ []string) map[string]map[string]interface{} {
+func (e *retrievalCountEngine) GetFields(chunks []map[string]interface{}, selected []string) map[string]map[string]interface{} {
fields := make(map[string]map[string]interface{}, len(chunks))
for _, chunk := range chunks {
if id, ok := chunk["id"].(string); ok {
- fields[id] = chunk
+ projected := make(map[string]interface{}, len(selected))
+ for _, field := range selected {
+ if value, ok := chunk[field]; ok {
+ projected[field] = value
+ }
+ }
+ fields[id] = projected
}
}
return fields