RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: exclude disabled documents from Wiki products (#20154)
buua4366fec6bba593e
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 2cc3c1127f..43c2a4df9c 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -26,6 +26,7 @@ import ("ragflow/internal/common""ragflow/internal/engine""ragflow/internal/engine/types"+ "ragflow/internal/entity"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")@@ -69,6 +70,25 @@ type documentWikiPageReader interface {LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant, kb string, slugs []string) ([]kccommon.Product, error)}+// enabledDocumentIDs returns the documents that may contribute to dataset-level+// Wiki products. Wiki staging rows intentionally keep available_int=0 even when+// their document is disabled, so availability alone cannot identify active+// contributions.+//+// The bool reports whether the database is configured. Offline/unit readers do+// not have document status available and retain the previous engine-only behavior.+func enabledDocumentIDs(ctx context.Context, datasetID string) ([]string, bool, error) {+ if kcDB == nil {+ return nil, false, nil+ }+ var ids []string+ err := kcDB.WithContext(ctx).+ Model(&entity.Document{}).+ Where("kb_id = ? AND (status IS NULL OR status <> ?)", datasetID, "0").+ Pluck("id", &ids).Error+ return ids, true, err+}+// engineReader loads the per-document compiled products through the global// DocEngine (§11.6 step 1, §11.7 incremental re-dedup). It depends on the// process-wide DocEngine obtained via engine.Get(); the engine abstraction owns@@ -268,11 +288,21 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,if eng == nil || len(slugs) == 0 {return nil, nil}+ enabledIDs, statusAvailable, err := enabledDocumentIDs(ctx, kb)+ if err != nil {+ return nil, fmt.Errorf("load enabled Wiki documents: %w", err)+ }+ if statusAvailable && len(enabledIDs) == 0 {+ return nil, nil+ }filter := map[string]interface{}{"available_int": 0,"compile_kwd": compileKwdWikiPage,"slug_kwd": slugs,}+ if statusAvailable {+ filter["doc_id"] = enabledIDs+ }var pages []kccommon.Productfor offset := 0; ; offset += loadDocProductsLimit {res, err := eng.Search(ctx, &types.SearchRequest{diff --git a/internal/ingestion/knowledge_compile/reader_test.go b/internal/ingestion/knowledge_compile/reader_test.goindex 9642e1a89d..17da6c06e3 100644--- a/internal/ingestion/knowledge_compile/reader_test.go+++ b/internal/ingestion/knowledge_compile/reader_test.go@@ -17,11 +17,62 @@ package knowledge_compileimport ("context"+ "reflect""testing"+ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/entity"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")+func TestWikiReadersFilterDisabledDocuments(t *testing.T) {+ db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err := db.AutoMigrate(&entity.Document{}); err != nil {+ t.Fatalf("migrate documents: %v", err)+ }+ activeStatus := "1"+ disabledStatus := "0"+ for _, doc := range []*entity.Document{+ {ID: "active-doc", KbID: "kb1", Status: &activeStatus, ParserConfig: entity.JSONMap{}},+ {ID: "disabled-doc", KbID: "kb1", Status: &disabledStatus, ParserConfig: entity.JSONMap{}},+ } {+ if err := db.Create(doc).Error; err != nil {+ t.Fatalf("create document %s: %v", doc.ID, err)+ }+ }+ previousDB := kcDB+ kcDB = db+ t.Cleanup(func() { kcDB = previousDB })++ eng := &fakeEngine{searchChunks: []map[string]interface{}{{+ "id": "page-1", "doc_id": "active-doc", "compile_kwd": "wiki_page",+ "available_int": 0, "kc_payload": "page content", "slug_kwd": "entity/page",+ "page_type_kwd": "entity", "title_kwd": "page", "source_doc_ids": []string{"active-doc"},+ }}}+ r := engineReader{eng: eng}+ if _, err := r.LoadDocumentWikiPagesBySlugs(t.Context(), "tenant", "kb1", []string{"entity/page"}); err != nil {+ t.Fatalf("load document Wiki pages: %v", err)+ }+ wikiFilter, ok := eng.lastSearchReq.Filter["doc_id"]+ if !ok || !reflect.DeepEqual(wikiFilter, []string{"active-doc"}) {+ t.Fatalf("document Wiki filter = %#v, want active document IDs", wikiFilter)+ }++ w := engineWriter{eng: eng}+ if _, err := w.loadActiveDocumentWikiPages(t.Context(), "tenant", "kb1"); err != nil {+ t.Fatalf("load active document Wiki pages: %v", err)+ }+ graphFilter, ok := eng.lastSearchReq.Filter["doc_id"]+ if !ok || !reflect.DeepEqual(graphFilter, []string{"active-doc"}) {+ t.Fatalf("graph Wiki filter = %#v, want active document IDs", graphFilter)+ }+}+// TestSearchSimilarFiltersByVariant asserts the B1/KNN contract: SearchSimilar// scopes the engine query to available_int=1 AND compile_kwd=variant, and the// in-memory dirty-row guard drops any row whose compile_kwd does not map back todiff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.goindex bd1dd2a5f5..586b73ef06 100644--- a/internal/ingestion/knowledge_compile/structure_merge_test.go+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go@@ -11,10 +11,13 @@ import (// fakeWriter is a Writer double that records the last WriteMergedStructure// buckets so a test can assert the grouping behavior without an engine.type fakeWriter struct {- buckets []StructureBucket+ buckets []StructureBucket+ writeMergedCalls int+ projectWikiGraphCalls int}func (f *fakeWriter) WriteMerged(context.Context, string, string, []kccommon.Product) error {+ f.writeMergedCalls++return nil}func (f *fakeWriter) WriteMergedStructure(_ context.Context, _, _ string, buckets []StructureBucket) error {@@ -34,8 +37,11 @@ func (f *fakeWriter) DeleteDocLevelForDocs(context.Context, string, string, []stfunc (f *fakeWriter) StripMergedSources(context.Context, string, string, []string) error {return nil}-func (f *fakeWriter) ProjectWikiGraph(context.Context, string, string) error { return nil }-func (f *fakeWriter) DropWikiGraph(context.Context, string, string) error { return nil }+func (f *fakeWriter) ProjectWikiGraph(context.Context, string, string) error {+ f.projectWikiGraphCalls+++ return nil+}+func (f *fakeWriter) DropWikiGraph(context.Context, string, string) error { return nil }// TestMergeStructureDataset_GroupsByName covers G1: structure products are// bucketed by name, descriptions folded, source ids unioned, and onlydiff --git a/internal/ingestion/knowledge_compile/wiki_contribution_test.go b/internal/ingestion/knowledge_compile/wiki_contribution_test.goindex 5ba6d3143f..3a4c942991 100644--- a/internal/ingestion/knowledge_compile/wiki_contribution_test.go+++ b/internal/ingestion/knowledge_compile/wiki_contribution_test.go@@ -5,6 +5,7 @@ import ("testing"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/service/nav")func wikiContributionTestProduct(slug, content string, chunkIDs ...string) kccommon.Product {@@ -66,6 +67,38 @@ func TestDiffWikiDocumentContributionTreatsDisabledDocumentAsRetraction(t *testi}}+func TestProcessBatchRetractionWithoutWikiContributionSkipsMerge(t *testing.T) {+ store := &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}+ previousNav := nav.GetNavService()+ nav.SetNavService(&recordingNavService{})+ t.Cleanup(func() { nav.SetNavService(previousNav) })+ fw := &fakeWriter{}+ factoryCalled := false+ consumer := NewConsumer(NewFakeScheduler(),+ WithReader(&fakeReader{}),+ WithWriter(fw),+ withWikiContributionStore(store),+ WithDeduperFactory(func(string) (Deduper, error) {+ factoryCalled = true+ return nil, nil+ }),+ )+ if err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "", []BacklogEntry{{+ DocID: "doc-1", EventType: string(EventTypeDisabled), Variants: []string{"wiki"},+ }}, []string{kccommon.TaskTypeWiki}); err != nil {+ t.Fatalf("processBatch failed: %v", err)+ }+ if factoryCalled {+ t.Fatal("retraction-only Wiki event initialized the dataset merge path")+ }+ if fw.writeMergedCalls != 0 {+ t.Fatalf("retraction-only Wiki event wrote %d merged product batches", fw.writeMergedCalls)+ }+ if fw.projectWikiGraphCalls != 0 {+ t.Fatalf("retraction without a Wiki contribution projected graph %d times, want 0", fw.projectWikiGraphCalls)+ }+}+func TestProcessBatchSkipsUnchangedWikiOnlyEvent(t *testing.T) {product := wikiContributionTestProduct("cao-cao", "same content", "chunk-1")store := &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex d7f1aa3084..b5a13744c6 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -1478,16 +1478,27 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, kif eng == nil {return nil, nil}+ enabledIDs, statusAvailable, err := enabledDocumentIDs(ctx, kb)+ if err != nil {+ return nil, fmt.Errorf("load enabled Wiki documents for graph: %w", err)+ }+ if statusAvailable && len(enabledIDs) == 0 {+ return nil, nil+ }const batchSize = 2000bySlug := make(map[string]wikiPageProjection)for offset := 0; ; offset += batchSize {+ filter := map[string]interface{}{+ "compile_kwd": compileKwdWikiPage, "available_int": 0,+ "scope_kwd": "doc", "kb_id": kb,+ }+ if statusAvailable {+ filter["doc_id"] = enabledIDs+ }result, err := eng.Search(ctx, &types.SearchRequest{IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)},KbIDs: []string{kb},- Filter: map[string]interface{}{- "compile_kwd": compileKwdWikiPage, "available_int": 0,- "scope_kwd": "doc", "kb_id": kb,- },+ Filter: filter,SelectFields: []string{"slug_kwd", "page_type_kwd", "title_kwd", "entity_names_kwd","summary_with_weight", "outlinks_kwd", "source_doc_ids", "source_chunk_ids",diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex 120e7b614c..04c0dd8893 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -93,7 +93,10 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,continue}}- s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, docID)+ // A status transition changes document availability, not its parsed+ // content. The status event handles dataset-level retraction/re-enable;+ // scheduling a delayed Wiki recompilation here creates duplicate events+ // and a visible 20-second oscillation.s.publishKnowledgeCompileStatusChange(ctx, kb.TenantID, doc.KbID, docID, statusInt)result[docID] = map[string]string{"status": status}}diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex 6c8eb0676b..002d1bf080 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -592,7 +592,10 @@ func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *entreturn err}}- s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, doc.ID)+ // A status transition changes document availability, not its parsed+ // content. The status event handles dataset-level retraction/re-enable;+ // scheduling a delayed Wiki recompilation here creates duplicate events+ // and a visible 20-second oscillation.s.publishKnowledgeCompileStatusChange(ctx, kb.TenantID, doc.KbID, doc.ID, status)return nil}diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex 13a520b243..3ee329dfca 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -2615,6 +2615,32 @@ func TestUpdateDocumentChunkAvailabilityTogglesFinalProducts(t *testing.T) {}}+func TestDocumentKnowledgeCompileTypesIncludesUnavailableProducts(t *testing.T) {+ docEngine := &sourceAvailabilityDocEngine{}+ svc := testDocumentService(t)+ svc.docEngine = docEngine++ variants, _, err := svc.documentKnowledgeCompileTypes(t.Context(), "tenant-1", "kb-1", "doc-1")+ if err != nil {+ t.Fatalf("documentKnowledgeCompileTypes failed: %v", err)+ }+ if docEngine.search == nil || !docEngine.search.IncludeUnavailable {+ t.Fatalf("knowledge compile type search = %#v, want unavailable products included", docEngine.search)+ }+ for _, want := range []string{"tree", "structure", "wiki"} {+ found := false+ for _, variant := range variants {+ if variant == want {+ found = true+ break+ }+ }+ if !found {+ t.Fatalf("variants = %v, missing %q", variants, want)+ }+ }+}+func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.goindex d300088edc..cda55970ec 100644--- a/internal/service/document/knowledge_compile_status.go+++ b/internal/service/document/knowledge_compile_status.go@@ -65,12 +65,13 @@ func (s *DocumentService) documentKnowledgeCompileTypes(ctx context.Context, tenfor offset := 0; ; offset += 1000 {searchCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)result, err := s.docEngine.Search(searchCtx, &types.SearchRequest{- IndexNames: []string{indexName},- KbIDs: []string{datasetID},- Offset: offset,- Limit: 1000,- SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd"},- Filter: map[string]any{"doc_id": []string{documentID}},+ IndexNames: []string{indexName},+ KbIDs: []string{datasetID},+ Offset: offset,+ Limit: 1000,+ SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd"},+ Filter: map[string]any{"doc_id": []string{documentID}},+ IncludeUnavailable: true,})cancel()if err != nil {