RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(nlp): add retrieval prune logic and tests (#21627)
ZZZZY6fd888fc0b82
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex 32c0b6b5e0..a82279144e 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -1233,27 +1233,24 @@ func RetrievalByChildren(chunks []map[string]interface{}, tenantIDs []string, doreturn remainingChunks}-// PruneDeletedChunks removes chunks whose documents no longer exist+// PruneDeletedChunks keeps source-backed chunks and dataset compilation chunks.func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *RetrievalSearchResult) (*RetrievalSearchResult, error) {if s.documentDAO == nil {return nil, fmt.Errorf("documentDAO is not initialized")}- // Collect all doc_ids from chunks- chunkDocIDs := make([]string, 0, len(result.Field))- for _, chunk := range result.Field {- if isDatasetCompilationChunk(chunk) {+ // Collect source document IDs from the ordered candidates.+ chunkDocIDs := make([]string, 0, len(result.IDs))+ for _, chunkID := range result.IDs {+ chunk := result.Field[chunkID]+ if len(chunk) == 0 || isDatasetCompilationChunk(chunk) {continue}- if docID, ok := chunk["doc_id"].(string); ok && docID != "" {+ if docID := strings.TrimSpace(firstStringValue(chunk["doc_id"])); docID != "" {chunkDocIDs = append(chunkDocIDs, docID)}}- if len(chunkDocIDs) == 0 {- return result, nil- }-- // Deduplicate chunkDocIDs for correct comparison with existingDocIDs+ // Deduplicate source IDs before the batch lookup.uniqueDocIDs := make([]string, 0, len(chunkDocIDs))seen := make(map[string]struct{}, len(chunkDocIDs))for _, id := range chunkDocIDs {@@ -1274,42 +1271,26 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *RetriexistingDocIDs[doc.ID] = struct{}{}}- // Early return if all docs exist- if len(existingDocIDs) == len(uniqueDocIDs) {- return result, nil- }-- // Filter out chunks with deleted documents+ // Validate every candidate, including ghosts and chunks without a source ID.filteredIDs := make([]string, 0, len(result.IDs))- filteredChunks := make([]map[string]interface{}, 0, len(result.IDs))filteredField := make(map[string]map[string]interface{}, len(result.IDs))filteredHighlight := make(map[string]string)removed := 0for _, chunkID := range result.IDs {- chunk, exists := result.Field[chunkID]- if !exists {+ chunk := result.Field[chunkID]+ if len(chunk) == 0 {+ removed++continue}- docID, ok := chunk["doc_id"].(string)- if !ok || docID == "" || isDatasetCompilationChunk(chunk) {- // Dataset artifacts use the dataset ID instead of a source document ID.- filteredIDs = append(filteredIDs, chunkID)- filteredChunks = append(filteredChunks, chunk)- filteredField[chunkID] = chunk- if result.Highlight != nil {- if hl, ok := result.Highlight[chunkID]; ok {- filteredHighlight[chunkID] = hl- }+ if !isDatasetCompilationChunk(chunk) {+ docID := strings.TrimSpace(firstStringValue(chunk["doc_id"]))+ if _, docExists := existingDocIDs[docID]; docID == "" || !docExists {+ removed+++ continue}- continue- }- if _, docExists := existingDocIDs[docID]; !docExists {- removed++- continue}filteredIDs = append(filteredIDs, chunkID)- filteredChunks = append(filteredChunks, chunk)filteredField[chunkID] = chunkif result.Highlight != nil {if hl, ok := result.Highlight[chunkID]; ok {@@ -1318,10 +1299,27 @@ func (s *RetrievalService) PruneDeletedChunks(ctx context.Context, result *Retri}}- if removed > 0 {- common.Warn("Pruned stale chunks whose documents no longer exist", zap.Int("removed", removed))+ if removed == 0 {+ return result, nil+ }++ // Preserve raw engine fields needed by KNN and local scoring.+ chunksByID := make(map[string]map[string]interface{}, len(result.Chunks))+ for _, chunk := range result.Chunks {+ ids := s.docEngine.GetChunkIDs([]map[string]interface{}{chunk})+ if len(ids) == 1 {+ chunksByID[ids[0]] = chunk+ }+ }+ filteredChunks := make([]map[string]interface{}, 0, len(filteredIDs))+ for _, chunkID := range filteredIDs {+ if chunk, ok := chunksByID[chunkID]; ok {+ filteredChunks = append(filteredChunks, chunk)+ }}+ common.Warn("Pruned chunks without an existing source document", zap.Int("removed", removed))+return &RetrievalSearchResult{Chunks: filteredChunks,Total: int64(len(filteredIDs)),diff --git a/internal/service/nlp/retrieval_prune_test.go b/internal/service/nlp/retrieval_prune_test.gonew file mode 100644index 0000000000..ff260daeec--- /dev/null+++ b/internal/service/nlp/retrieval_prune_test.go@@ -0,0 +1,193 @@+package nlp++import (+ "slices"+ "testing"++ "ragflow/internal/dao"+ "ragflow/internal/entity"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"+)++func TestIsDatasetCompilationChunk(t *testing.T) {+ for _, tt := range []struct {+ name string+ chunk map[string]interface{}+ want bool+ }{+ {+ name: "complete dataset compilation identity",+ chunk: map[string]interface{}{"scope_kwd": "dataset", "compile_kwd": "graph", "doc_id": "kb-1", "kb_id": "kb-1"},+ want: true,+ },+ {name: "ordinary chunk without doc_id", chunk: map[string]interface{}{"kb_id": "kb-1"}},+ {name: "ordinary chunk with empty doc_id", chunk: map[string]interface{}{"doc_id": "", "kb_id": "kb-1"}},+ {name: "compilation without doc_id", chunk: map[string]interface{}{"scope_kwd": "dataset", "compile_kwd": "graph", "kb_id": "kb-1"}},+ {name: "sentinel without compilation marker", chunk: map[string]interface{}{"scope_kwd": "dataset", "doc_id": "kb-1", "kb_id": "kb-1"}},+ {name: "sentinel without dataset scope", chunk: map[string]interface{}{"compile_kwd": "graph", "doc_id": "kb-1", "kb_id": "kb-1"}},+ {name: "list-wrapped sentinel", chunk: map[string]interface{}{"scope_kwd": "dataset", "compile_kwd": "graph", "doc_id": []string{"kb-1"}, "kb_id": []string{"kb-1"}}},+ {name: "document compilation", chunk: map[string]interface{}{"scope_kwd": "doc", "compile_kwd": "graph", "doc_id": "doc-1", "kb_id": "kb-1"}},+ } {+ t.Run(tt.name, func(t *testing.T) {+ if got := isDatasetCompilationChunk(tt.chunk); got != tt.want {+ t.Fatalf("isDatasetCompilationChunk(%v) = %v, want %v", tt.chunk, got, tt.want)+ }+ })+ }+}++func TestPruneDeletedChunksFiltersOrphans(t *testing.T) {+ newRetrievalTestDB(t, "doc-live")+ liveA := map[string]interface{}{"id": "live-a", "doc_id": "doc-live", "kb_id": "kb-1", "_index": "ragflow_tenant-1"}+ liveB := map[string]interface{}{"id": "live-b", "doc_id": "doc-live", "kb_id": "kb-1", "_index": "ragflow_tenant-1"}+ missing := map[string]interface{}{"id": "missing", "kb_id": "kb-1"}+ empty := map[string]interface{}{"id": "empty", "doc_id": "", "kb_id": "kb-1"}+ compiledWithoutSource := map[string]interface{}{"id": "compiled", "kb_id": "kb-1", "compile_kwd": "graph", "scope_kwd": "dataset"}+ docEngine := &retrievalCountEngine{}+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})++ for _, tt := range []struct {+ name string+ ids []string+ chunks []map[string]interface{}+ want []string+ }{+ {+ name: "live sources mixed with missing and empty doc_id",+ ids: []string{"live-b", "missing", "live-a", "empty"},+ chunks: []map[string]interface{}{liveA, missing, empty, liveB},+ want: []string{"live-b", "live-a"},+ },+ {+ name: "all ordinary chunks lack a source",+ ids: []string{"missing", "empty"},+ chunks: []map[string]interface{}{missing, empty},+ },+ {+ name: "compilation marker cannot preserve chunks without a source",+ ids: []string{"missing", "compiled", "empty"},+ chunks: []map[string]interface{}{missing, compiledWithoutSource, empty},+ },+ {+ name: "ghost is removed even when every source exists",+ ids: []string{"live-a", "ghost"},+ chunks: []map[string]interface{}{liveA},+ want: []string{"live-a"},+ },+ } {+ t.Run(tt.name, func(t *testing.T) {+ highlight := make(map[string]string, len(tt.ids))+ for _, id := range tt.ids {+ highlight[id] = "highlight " + id+ }+ result := &RetrievalSearchResult{+ IDs: tt.ids,+ Chunks: tt.chunks,+ Field: docEngine.GetFields(tt.chunks, []string{"doc_id", "kb_id", "compile_kwd", "scope_kwd"}),+ Highlight: highlight,+ Total: int64(len(tt.ids)),+ }+ pruned, err := service.PruneDeletedChunks(t.Context(), result)+ if err != nil {+ t.Fatal(err)+ }+ if !slices.Equal(pruned.IDs, tt.want) || pruned.Total != int64(len(tt.want)) {+ t.Fatalf("IDs = %v, total = %d; want %v, %d", pruned.IDs, pruned.Total, tt.want, len(tt.want))+ }+ if len(pruned.Field) != len(tt.want) || len(pruned.Chunks) != len(tt.want) || len(pruned.Highlight) != len(tt.want) {+ t.Fatalf("field/chunks/highlight sizes = %d/%d/%d, want %d", len(pruned.Field), len(pruned.Chunks), len(pruned.Highlight), len(tt.want))+ }+ for i, id := range tt.want {+ if _, ok := pruned.Field[id]; !ok {+ t.Fatalf("retained chunk %q is missing from Field", id)+ }+ if chunk := pruned.Chunks[i]; chunk["id"] != id || chunk["_index"] != "ragflow_tenant-1" {+ t.Fatalf("raw chunk at %d = %v, want %q with its original index", i, chunk, id)+ }+ if got := pruned.Highlight[id]; got != "highlight "+id {+ t.Fatalf("highlight for %q = %q", id, got)+ }+ }+ })+ }+}++func TestPruneDeletedChunksSkipsLookupForDatasetCompilation(t *testing.T) {+ for _, tt := range []struct {+ name string+ ids []string+ }{+ {name: "dataset compilation only", ids: []string{"compiled"}},+ {name: "dataset compilation with ghost", ids: []string{"compiled", "ghost"}},+ } {+ t.Run(tt.name, func(t *testing.T) {+ db := newRetrievalTestDB(t)+ queries := 0+ if err := db.Callback().Query().Before("gorm:query").Register("count_prune_document_queries", func(*gorm.DB) {+ queries+++ }); err != nil {+ t.Fatal(err)+ }+ compiled := map[string]interface{}{"id": "compiled", "doc_id": "kb-1", "kb_id": "kb-1", "compile_kwd": "graph", "scope_kwd": "dataset", "_index": "ragflow_tenant-1"}+ docEngine := &retrievalCountEngine{}+ chunks := []map[string]interface{}{compiled}+ result := &RetrievalSearchResult{+ IDs: tt.ids,+ Chunks: chunks,+ Field: docEngine.GetFields(chunks, []string{"doc_id", "kb_id", "compile_kwd", "scope_kwd"}),+ Highlight: map[string]string{"compiled": "compiled highlight"},+ Total: int64(len(tt.ids)),+ }+ if len(tt.ids) > 1 {+ result.Highlight["ghost"] = "ghost highlight"+ }+ pruned, err := NewRetrievalService(docEngine, &dao.DocumentDAO{}).PruneDeletedChunks(t.Context(), result)+ if err != nil {+ t.Fatal(err)+ }+ if queries != 0 {+ t.Fatalf("document queries = %d, want 0", queries)+ }+ if !slices.Equal(pruned.IDs, []string{"compiled"}) || pruned.Total != 1 {+ t.Fatalf("IDs = %v, total = %d; want [compiled], 1", pruned.IDs, pruned.Total)+ }+ if len(pruned.Field) != 1 || len(pruned.Chunks) != 1 || len(pruned.Highlight) != 1 {+ t.Fatalf("field/chunks/highlight sizes = %d/%d/%d, want 1", len(pruned.Field), len(pruned.Chunks), len(pruned.Highlight))+ }+ if _, ok := pruned.Field["compiled"]; !ok {+ t.Fatal("compiled chunk is missing from Field")+ }+ if pruned.Chunks[0]["id"] != "compiled" || pruned.Chunks[0]["_index"] != "ragflow_tenant-1" || pruned.Highlight["compiled"] != "compiled highlight" {+ t.Fatalf("compiled raw chunk or highlight was not preserved: %v, %v", pruned.Chunks, pruned.Highlight)+ }+ })+ }+}++func newRetrievalTestDB(t *testing.T, docIDs ...string) *gorm.DB {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ if err != nil {+ t.Fatal(err)+ }+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatal(err)+ }+ sqlDB.SetMaxOpenConns(1)+ t.Cleanup(func() { _ = sqlDB.Close() })+ if err := db.AutoMigrate(&entity.Document{}); err != nil {+ t.Fatal(err)+ }+ for _, id := range docIDs {+ if err := db.Create(&entity.Document{ID: id, KbID: "kb-1", ParserConfig: entity.JSONMap{}}).Error; err != nil {+ t.Fatal(err)+ }+ }+ oldDB := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = oldDB })+ return db+}diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 3761e34587..b35f71b7a9 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -21,6 +21,7 @@ import ()func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")for _, tc := range []struct {name stringpage int@@ -63,6 +64,7 @@ func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {for i := range rows {rows[i] = map[string]interface{}{"id": fmt.Sprintf("chunk-%02d", i),+ "doc_id": "doc-1","content_ltks": "alpha","content_with_weight": "alpha","_score": 0.9,@@ -405,13 +407,14 @@ func TestRetrievalRerankCandidatesGuardPrecedesPaginationGuard(t *testing.T) {}func TestRetrievalFirstPageWithRerankModelCallsReranker(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()driver := &captureDriver{}docEngine := &retrievalCountEngine{rows: []map[string]interface{}{- {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},+ {"id": "chunk-1", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},}}service := NewRetrievalService(docEngine, &dao.DocumentDAO{})top := 10@@ -439,13 +442,14 @@ func TestRetrievalFirstPageWithRerankModelCallsReranker(t *testing.T) {}func TestRetrievalPreservesCompilationKindWhenPresent(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()docEngine := &retrievalCountEngine{rows: []map[string]interface{}{- {"id": "compiled", "content_ltks": "alpha", "content_with_weight": "alpha", "compile_kwd": "page_index", "_score": 0.9},- {"id": "ordinary", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},+ {"id": "compiled", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "compile_kwd": "page_index", "_score": 0.9},+ {"id": "ordinary", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},}}service := NewRetrievalService(docEngine, &dao.DocumentDAO{})threshold := 0.0@@ -483,12 +487,13 @@ func TestRetrievalPreservesCompilationKindWhenPresent(t *testing.T) {// reranker reads pagerank_fea and tag_feas off Field, and Field is the engine's// projection of exactly SelectFields, so both must be requested.func TestRetrievalProjectsRankFeatureFields(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()docEngine := &retrievalCountEngine{rows: []map[string]interface{}{- {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},+ {"id": "chunk-1", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},}}service := NewRetrievalService(docEngine, &dao.DocumentDAO{})threshold := 0.0@@ -521,12 +526,13 @@ func TestRetrievalProjectsRankFeatureFields(t *testing.T) {// "questions", but both are read off Field — the engine's projection of exactly// SelectFields — so omitting them scores those tokens as absent.func TestRetrievalProjectsQuestionTokenFields(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()docEngine := &retrievalCountEngine{rows: []map[string]interface{}{- {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "question_tks": "beta", "_score": 0.9},+ {"id": "chunk-1", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "question_tks": "beta", "_score": 0.9},}}service := NewRetrievalService(docEngine, &dao.DocumentDAO{})threshold := 0.0@@ -557,13 +563,14 @@ func TestRetrievalProjectsQuestionTokenFields(t *testing.T) {// only pagerank_fea differs, so the boosted chunk must win and its similarity// must carry the pagerank on top of the hybrid score.func TestRetrievalPagerankBoostsRanking(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()docEngine := &retrievalCountEngine{rows: []map[string]interface{}{- {"id": "plain", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5},- {"id": "boosted", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(4)},+ {"id": "plain", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5},+ {"id": "boosted", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(4)},}}service := NewRetrievalService(docEngine, &dao.DocumentDAO{})threshold := 0.0@@ -599,12 +606,13 @@ func TestRetrievalPagerankBoostsRanking(t *testing.T) {// rank_feature clause and the similarity must pick pagerank up locally.// vector_similarity keeps the engine's raw kNN score (asserted below).func TestRetrievalVectorOnlyKeepsPagerank(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()docEngine := &retrievalCountEngine{rows: []map[string]interface{}{- {"id": "chunk-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(3)},+ {"id": "chunk-1", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.5, common.PAGERANK_FLD: float64(3)},}}service := NewRetrievalService(docEngine, &dao.DocumentDAO{})threshold := 0.0@@ -638,13 +646,14 @@ func TestRetrievalVectorOnlyKeepsPagerank(t *testing.T) {}func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()rows := []map[string]interface{}{- {"id": "highlighted", "content_ltks": "alpha", "content_with_weight": "alpha élève", "_score": 0.9},- {"id": "unmatched", "content_ltks": "alpha", "content_with_weight": "plain content", "_score": 0.8},+ {"id": "highlighted", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha élève", "_score": 0.9},+ {"id": "unmatched", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "plain content", "_score": 0.8},}docEngine := &retrievalCountEngine{rows: rows,@@ -1345,13 +1354,14 @@ func (e *perIndexEngine) Search(ctx context.Context, req *types.SearchRequest) (}func TestRetrievalSearchesEachTenantIndexOnce(t *testing.T) {+ newRetrievalTestDB(t, "doc-1")oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()docEngine := &perIndexEngine{retrievalCountEngine: retrievalCountEngine{rows: []map[string]interface{}{- {"id": "chunk-a", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},- {"id": "chunk-b", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},+ {"id": "chunk-a", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.9},+ {"id": "chunk-b", "doc_id": "doc-1", "content_ltks": "alpha", "content_with_weight": "alpha", "_score": 0.8},}}}service := NewRetrievalService(docEngine, &dao.DocumentDAO{})threshold := 0.0