Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(nlp): add retrieval prune logic and tests (#21627)

ZZZZY6fd888fc0b82
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index 32c0b6b5e0..a82279144e 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -1233,27 +1233,24 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, do
return remainingChunks
}
-// PruneDeletedChunks removes chunks whose documents no longer exist
+// PruneDeletedChunks keeps source-backed chunks and dataset compilation chunks.
func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *RetrievalSearchResult) (*RetrievalSearchResult, error) {
if s.documentDAO == nil {
return nil, fmt.Errorf("documentDAO is not initialized")
}
- // Collect all doc_ids from chunks
- chunkDocIDs := make([]string, 0, len(result.Field))
- for _, chunk := range result.Field {
- if isDatasetCompilationChunk(chunk) {
+ // Collect source document IDs from the ordered candidates.
+ chunkDocIDs := make([]string, 0, len(result.IDs))
+ for _, chunkID := range result.IDs {
+ chunk := result.Field[chunkID]
+ if len(chunk) == 0 || isDatasetCompilationChunk(chunk) {
continue
}
- if docID, ok := chunk["doc_id"].(string); ok && docID != "" {
+ if docID := strings.TrimSpace(firstStringValue(chunk["doc_id"])); docID != "" {
chunkDocIDs = append(chunkDocIDs, docID)
}
}
- if len(chunkDocIDs) == 0 {
- return result, nil
- }
-
- // Deduplicate chunkDocIDs for correct comparison with existingDocIDs
+ // Deduplicate source IDs before the batch lookup.
uniqueDocIDs := make([]string, 0, len(chunkDocIDs))
seen := make(map[string]struct{}, len(chunkDocIDs))
for _, id := range chunkDocIDs {
@@ -1274,42 +1271,26 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *Retri
existingDocIDs[doc.ID] = struct{}{}
}
- // Early return if all docs exist
- if len(existingDocIDs) == len(uniqueDocIDs) {
- return result, nil
- }
-
- // Filter out chunks with deleted documents
+ // Validate every candidate, including ghosts and chunks without a source ID.
filteredIDs := make([]string, 0, len(result.IDs))
- filteredChunks := make([]map[string]interface{}, 0, len(result.IDs))
filteredField := make(map[string]map[string]interface{}, len(result.IDs))
filteredHighlight := make(map[string]string)
removed := 0
for _, chunkID := range result.IDs {
- chunk, exists := result.Field[chunkID]
- if !exists {
+ chunk := result.Field[chunkID]
+ if len(chunk) == 0 {
+ removed++
continue
}
- docID, ok := chunk["doc_id"].(string)
- if !ok || docID == "" || isDatasetCompilationChunk(chunk) {
- // Dataset artifacts use the dataset ID instead of a source document ID.
- filteredIDs = append(filteredIDs, chunkID)
- filteredChunks = append(filteredChunks, chunk)
- filteredField[chunkID] = chunk
- if result.Highlight != nil {
- if hl, ok := result.Highlight[chunkID]; ok {
- filteredHighlight[chunkID] = hl
- }
+ if !isDatasetCompilationChunk(chunk) {
+ docID := strings.TrimSpace(firstStringValue(chunk["doc_id"]))
+ if _, docExists := existingDocIDs[docID]; docID == "" || !docExists {
+ removed++
+ continue
}
- continue
- }
- if _, docExists := existingDocIDs[docID]; !docExists {
- removed++
- continue
}
filteredIDs = append(filteredIDs, chunkID)
- filteredChunks = append(filteredChunks, chunk)
filteredField[chunkID] = chunk
if result.Highlight != nil {
if hl, ok := result.Highlight[chunkID]; ok {
@@ -1318,10 +1299,27 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *Retri
}
}
- if removed > 0 {
- common.Warn("Pruned stale chunks whose documents no longer exist", zap.Int("removed", removed))
+ if removed == 0 {
+ return result, nil
+ }
+
+ // Preserve raw engine fields needed by KNN and local scoring.
+ chunksByID := make(map[string]map[string]interface{}, len(result.Chunks))
+ for _, chunk := range result.Chunks {
+ ids := s.docEngine.GetChunkIDs([]map[string]interface{}{chunk})
+ if len(ids) == 1 {
+ chunksByID[ids[0]] = chunk
+ }
+ }
+ filteredChunks := make([]map[string]interface{}, 0, len(filteredIDs))
+ for _, chunkID := range filteredIDs {
+ if chunk, ok := chunksByID[chunkID]; ok {
+ filteredChunks = append(filteredChunks, chunk)
+ }
}
+ common.Warn("Pruned chunks without an existing source document", zap.Int("removed", removed))
+
return &RetrievalSearchResult{
Chunks: filteredChunks,
Total: int64(len(filteredIDs)),
diff --git a/internal/service/nlp/retrieval_prune_test.go b/internal/service/nlp/retrieval_prune_test.go
new file mode 100644
index 0000000000..ff260daeec
--- /dev/null
+++ b/internal/service/nlp/retrieval_prune_test.go
@@ -0,0 +1,193 @@
+package nlp
+
+import (
+ "slices"
+ "testing"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+)
+
+func TestIsDatasetCompilationChunk(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ chunk map[string]interface{}
+ want bool
+ }{
+ {
+ name: "complete dataset compilation identity",
+ chunk: map[string]interface{}{"scope_kwd": "dataset", "compile_kwd": "graph", "doc_id": "kb-1", "kb_id": "kb-1"},
+ want: true,
+ },
+ {name: "ordinary chunk without doc_id", chunk: map[string]interface{}{"kb_id": "kb-1"}},
+ {name: "ordinary chunk with empty doc_id", chunk: map[string]interface{}{"doc_id": "", "kb_id": "kb-1"}},
+ {name: "compilation without doc_id", chunk: map[string]interface{}{"scope_kwd": "dataset", "compile_kwd": "graph", "kb_id": "kb-1"}},
+ {name: "sentinel without compilation marker", chunk: map[string]interface{}{"scope_kwd": "dataset", "doc_id": "kb-1", "kb_id": "kb-1"}},
+ {name: "sentinel without dataset scope", chunk: map[string]interface{}{"compile_kwd": "graph", "doc_id": "kb-1", "kb_id": "kb-1"}},
+ {name: "list-wrapped sentinel", chunk: map[string]interface{}{"scope_kwd": "dataset", "compile_kwd": "graph", "doc_id": []string{"kb-1"}, "kb_id": []string{"kb-1"}}},
+ {name: "document compilation", chunk: map[string]interface{}{"scope_kwd": "doc", "compile_kwd": "graph", "doc_id": "doc-1", "kb_id": "kb-1"}},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := isDatasetCompilationChunk(tt.chunk); got != tt.want {
+ t.Fatalf("isDatasetCompilationChunk(%v) = %v, want %v", tt.chunk, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestPruneDeletedChunksFiltersOrphans(t *testing.T) {
+ newRetrievalTestDB(t, "doc-live")
+ liveA := map[string]interface{}{"id": "live-a", "doc_id": "doc-live", "kb_id": "kb-1", "_index": "ragflow_tenant-1"}
+ liveB := map[string]interface{}{"id": "live-b", "doc_id": "doc-live", "kb_id": "kb-1", "_index": "ragflow_tenant-1"}
+ missing := map[string]interface{}{"id": "missing", "kb_id": "kb-1"}
+ empty := map[string]interface{}{"id": "empty", "doc_id": "", "kb_id": "kb-1"}
+ compiledWithoutSource := map[string]interface{}{"id": "compiled", "kb_id": "kb-1", "compile_kwd": "graph", "scope_kwd": "dataset"}
+ docEngine := &retrievalCountEngine{}
+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
+
+ for _, tt := range []struct {
+ name string
+ ids []string
+ chunks []map[string]interface{}
+ want []string
+ }{
+ {
+ name: "live sources mixed with missing and empty doc_id",
+ ids: []string{"live-b", "missing", "live-a", "empty"},
+ chunks: []map[string]interface{}{liveA, missing, empty, liveB},
+ want: []string{"live-b", "live-a"},
+ },
+ {
+ name: "all ordinary chunks lack a source",
+ ids: []string{"missing", "empty"},
+ chunks: []map[string]interface{}{missing, empty},
+ },
+ {
+ name: "compilation marker cannot preserve chunks without a source",
+ ids: []string{"missing", "compiled", "empty"},
+ chunks: []map[string]interface{}{missing, compiledWithoutSource, empty},
+ },
+ {
+ name: "ghost is removed even when every source exists",
+ ids: []string{"live-a", "ghost"},
+ chunks: []map[string]interface{}{liveA},
+ want: []string{"live-a"},
+ },
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ highlight := make(map[string]string, len(tt.ids))
+ for _, id := range tt.ids {
+ highlight[id] = "highlight " + id
+ }
+ result := &RetrievalSearchResult{
+ IDs: tt.ids,
+ Chunks: tt.chunks,
+ Field: docEngine.GetFields(tt.chunks, []string{"doc_id", "kb_id", "compile_kwd", "scope_kwd"}),
+ Highlight: highlight,
+ Total: int64(len(tt.ids)),
+ }
+ pruned, err := service.PruneDeletedChunks(t.Context(), result)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Equal(pruned.IDs, tt.want) || pruned.Total != int64(len(tt.want)) {
+ t.Fatalf("IDs = %v, total = %d; want %v, %d", pruned.IDs, pruned.Total, tt.want, len(tt.want))
+ }
+ if len(pruned.Field) != len(tt.want) || len(pruned.Chunks) != len(tt.want) || len(pruned.Highlight) != len(tt.want) {
+ t.Fatalf("field/chunks/highlight sizes = %d/%d/%d, want %d", len(pruned.Field), len(pruned.Chunks), len(pruned.Highlight), len(tt.want))
+ }
+ for i, id := range tt.want {
+ if _, ok := pruned.Field[id]; !ok {
+ t.Fatalf("retained chunk %q is missing from Field", id)
+ }
+ if chunk := pruned.Chunks[i]; chunk["id"] != id || chunk["_index"] != "ragflow_tenant-1" {
+ t.Fatalf("raw chunk at %d = %v, want %q with its original index", i, chunk, id)
+ }
+ if got := pruned.Highlight[id]; got != "highlight "+id {
+ t.Fatalf("highlight for %q = %q", id, got)
+ }
+ }
+ })
+ }
+}
+
+func TestPruneDeletedChunksSkipsLookupForDatasetCompilation(t *testing.T) {
+ for _, tt := range []struct {
+ name string
+ ids []string
+ }{
+ {name: "dataset compilation only", ids: []string{"compiled"}},
+ {name: "dataset compilation with ghost", ids: []string{"compiled", "ghost"}},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ db := newRetrievalTestDB(t)
+ queries := 0
+ if err := db.Callback().Query().Before("gorm:query").Register("count_prune_document_queries", func(*gorm.DB) {
+ queries++
+ }); err != nil {
+ t.Fatal(err)
+ }
+ compiled := map[string]interface{}{"id": "compiled", "doc_id": "kb-1", "kb_id": "kb-1", "compile_kwd": "graph", "scope_kwd": "dataset", "_index": "ragflow_tenant-1"}
+ docEngine := &retrievalCountEngine{}
+ chunks := []map[string]interface{}{compiled}
+ result := &RetrievalSearchResult{
+ IDs: tt.ids,
+ Chunks: chunks,
+ Field: docEngine.GetFields(chunks, []string{"doc_id", "kb_id", "compile_kwd", "scope_kwd"}),
+ Highlight: map[string]string{"compiled": "compiled highlight"},
+ Total: int64(len(tt.ids)),
+ }
+ if len(tt.ids) > 1 {
+ result.Highlight["ghost"] = "ghost highlight"
+ }
+ pruned, err := NewRetrievalService(docEngine, &dao.DocumentDAO{}).PruneDeletedChunks(t.Context(), result)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if queries != 0 {
+ t.Fatalf("document queries = %d, want 0", queries)
+ }
+ if !slices.Equal(pruned.IDs, []string{"compiled"}) || pruned.Total != 1 {
+ t.Fatalf("IDs = %v, total = %d; want [compiled], 1", pruned.IDs, pruned.Total)
+ }
+ if len(pruned.Field) != 1 || len(pruned.Chunks) != 1 || len(pruned.Highlight) != 1 {
+ t.Fatalf("field/chunks/highlight sizes = %d/%d/%d, want 1", len(pruned.Field), len(pruned.Chunks), len(pruned.Highlight))
+ }
+ if _, ok := pruned.Field["compiled"]; !ok {
+ t.Fatal("compiled chunk is missing from Field")
+ }
+ if pruned.Chunks[0]["id"] != "compiled" || pruned.Chunks[0]["_index"] != "ragflow_tenant-1" || pruned.Highlight["compiled"] != "compiled highlight" {
+ t.Fatalf("compiled raw chunk or highlight was not preserved: %v, %v", pruned.Chunks, pruned.Highlight)
+ }
+ })
+ }
+}
+
+func newRetrievalTestDB(t *testing.T, docIDs ...string) *gorm.DB {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatal(err)
+ }
+ sqlDB.SetMaxOpenConns(1)
+ t.Cleanup(func() { _ = sqlDB.Close() })
+ if err := db.AutoMigrate(&entity.Document{}); err != nil {
+ t.Fatal(err)
+ }
+ for _, id := range docIDs {
+ if err := db.Create(&entity.Document{ID: id, KbID: "kb-1", ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatal(err)
+ }
+ }
+ oldDB := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = oldDB })
+ return db
+}
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index 3761e34587..b35f71b7a9 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -21,6 +21,7 @@ import (
)
func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
for _, tc := range []struct {
name string
page int
@@ -63,6 +64,7 @@ func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {
for i := range rows {
rows[i] = map[string]interface{}{
"id": fmt.Sprintf("chunk-%02d", i),
+ "doc_id": "doc-1",
"content_ltks": "alpha",
"content_with_weight": "alpha",
"_score": 0.9,
@@ -405,13 +407,14 @@ func TestRetrievalRerankCandidatesGuardPrecedesPaginationGuard(t *testing.T) {
}
func TestRetrievalFirstPageWithRerankModelCallsReranker(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
defer func() { globalQueryBuilder = oldQueryBuilder }()
driver := &captureDriver{}
docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
- {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},
+ {"id": "chunk-1", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},
}}
service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
top := 10
@@ -439,13 +442,14 @@ func TestRetrievalFirstPageWithRerankModelCallsReranker(t *testing.T) {
}
func TestRetrievalPreservesCompilationKindWhenPresent(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
defer func() { globalQueryBuilder = oldQueryBuilder }()
docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
- {"id": "compiled", "content_ltks": "alpha", "content_with_weight": "alpha", "compile_kwd": "page_index", "_score": 0.9},
- {"id": "ordinary", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},
+ {"id": "compiled", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "compile_kwd": "page_index", "_score": 0.9},
+ {"id": "ordinary", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},
}}
service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
threshold := 0.0
@@ -483,12 +487,13 @@ func TestRetrievalPreservesCompilationKindWhenPresent(t *testing.T) {
// reranker reads pagerank_fea and tag_feas off Field, and Field is the engine's
// projection of exactly SelectFields, so both must be requested.
func TestRetrievalProjectsRankFeatureFields(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
defer func() { globalQueryBuilder = oldQueryBuilder }()
docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
- {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},
+ {"id": "chunk-1", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},
}}
service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
threshold := 0.0
@@ -521,12 +526,13 @@ func TestRetrievalProjectsRankFeatureFields(t *testing.T) {
// "questions", but both are read off Field — the engine's projection of exactly
// SelectFields — so omitting them scores those tokens as absent.
func TestRetrievalProjectsQuestionTokenFields(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
defer func() { globalQueryBuilder = oldQueryBuilder }()
docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
- {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "question_tks": "beta", "_score": 0.9},
+ {"id": "chunk-1", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "question_tks": "beta", "_score": 0.9},
}}
service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
threshold := 0.0
@@ -557,13 +563,14 @@ func TestRetrievalProjectsQuestionTokenFields(t *testing.T) {
// only pagerank_fea differs, so the boosted chunk must win and its similarity
// must carry the pagerank on top of the hybrid score.
func TestRetrievalPagerankBoostsRanking(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
defer func() { globalQueryBuilder = oldQueryBuilder }()
docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
- {"id": "plain", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5},
- {"id": "boosted", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(4)},
+ {"id": "plain", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5},
+ {"id": "boosted", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(4)},
}}
service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
threshold := 0.0
@@ -599,12 +606,13 @@ func TestRetrievalPagerankBoostsRanking(t *testing.T) {
// rank_feature clause and the similarity must pick pagerank up locally.
// vector_similarity keeps the engine's raw kNN score (asserted below).
func TestRetrievalVectorOnlyKeepsPagerank(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
defer func() { globalQueryBuilder = oldQueryBuilder }()
docEngine := &retrievalCountEngine{rows: []map[string]interface{}{
- {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(3)},
+ {"id": "chunk-1", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(3)},
}}
service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
threshold := 0.0
@@ -638,13 +646,14 @@ func TestRetrievalVectorOnlyKeepsPagerank(t *testing.T) {
}
func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
defer func() { globalQueryBuilder = oldQueryBuilder }()
rows := []map[string]interface{}{
- {"id": "highlighted", "content_ltks": "alpha", "content_with_weight": "alpha élève", "_score": 0.9},
- {"id": "unmatched", "content_ltks": "alpha", "content_with_weight": "plain content", "_score": 0.8},
+ {"id": "highlighted", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha élève", "_score": 0.9},
+ {"id": "unmatched", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "plain content", "_score": 0.8},
}
docEngine := &retrievalCountEngine{
rows: rows,
@@ -1345,13 +1354,14 @@ func (e *perIndexEngine) Search(ctx context.Context, req *types.SearchRequest) (
}
func TestRetrievalSearchesEachTenantIndexOnce(t *testing.T) {
+ newRetrievalTestDB(t, "doc-1")
oldQueryBuilder := globalQueryBuilder
globalQueryBuilder = NewQueryBuilder()
defer func() { globalQueryBuilder = oldQueryBuilder }()
docEngine := &perIndexEngine{retrievalCountEngine: retrievalCountEngine{rows: []map[string]interface{}{
- {"id": "chunk-a", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},
- {"id": "chunk-b", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},
+ {"id": "chunk-a", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},
+ {"id": "chunk-b", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},
}}}
service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
threshold := 0.0