Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: exclude disabled documents from Wiki products (#20154)

buua4366fec6bba593e
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index 2cc3c1127f..43c2a4df9c 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -26,6 +26,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
+ "ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
@@ -69,6 +70,25 @@ type documentWikiPageReader interface {
LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant, kb string, slugs []string) ([]kccommon.Product, error)
}
+// enabledDocumentIDs returns the documents that may contribute to dataset-level
+// Wiki products. Wiki staging rows intentionally keep available_int=0 even when
+// their document is disabled, so availability alone cannot identify active
+// contributions.
+//
+// The bool reports whether the database is configured. Offline/unit readers do
+// not have document status available and retain the previous engine-only behavior.
+func enabledDocumentIDs(ctx context.Context, datasetID string) ([]string, bool, error) {
+ if kcDB == nil {
+ return nil, false, nil
+ }
+ var ids []string
+ err := kcDB.WithContext(ctx).
+ Model(&entity.Document{}).
+ Where("kb_id = ? AND (status IS NULL OR status <> ?)", datasetID, "0").
+ Pluck("id", &ids).Error
+ return ids, true, err
+}
+
// engineReader loads the per-document compiled products through the global
// DocEngine (§11.6 step 1, §11.7 incremental re-dedup). It depends on the
// process-wide DocEngine obtained via engine.Get(); the engine abstraction owns
@@ -268,11 +288,21 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,
if eng == nil || len(slugs) == 0 {
return nil, nil
}
+ enabledIDs, statusAvailable, err := enabledDocumentIDs(ctx, kb)
+ if err != nil {
+ return nil, fmt.Errorf("load enabled Wiki documents: %w", err)
+ }
+ if statusAvailable && len(enabledIDs) == 0 {
+ return nil, nil
+ }
filter := map[string]interface{}{
"available_int": 0,
"compile_kwd": compileKwdWikiPage,
"slug_kwd": slugs,
}
+ if statusAvailable {
+ filter["doc_id"] = enabledIDs
+ }
var pages []kccommon.Product
for offset := 0; ; offset += loadDocProductsLimit {
res, err := eng.Search(ctx, &types.SearchRequest{
diff --git a/internal/ingestion/knowledge_compile/reader_test.go b/internal/ingestion/knowledge_compile/reader_test.go
index 9642e1a89d..17da6c06e3 100644
--- a/internal/ingestion/knowledge_compile/reader_test.go
+++ b/internal/ingestion/knowledge_compile/reader_test.go
@@ -17,11 +17,62 @@ package knowledge_compile
import (
"context"
+ "reflect"
"testing"
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+
+ "ragflow/internal/entity"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
+func TestWikiReadersFilterDisabledDocuments(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.Document{}); err != nil {
+ t.Fatalf("migrate documents: %v", err)
+ }
+ activeStatus := "1"
+ disabledStatus := "0"
+ for _, doc := range []*entity.Document{
+ {ID: "active-doc", KbID: "kb1", Status: &activeStatus, ParserConfig: entity.JSONMap{}},
+ {ID: "disabled-doc", KbID: "kb1", Status: &disabledStatus, ParserConfig: entity.JSONMap{}},
+ } {
+ if err := db.Create(doc).Error; err != nil {
+ t.Fatalf("create document %s: %v", doc.ID, err)
+ }
+ }
+ previousDB := kcDB
+ kcDB = db
+ t.Cleanup(func() { kcDB = previousDB })
+
+ eng := &fakeEngine{searchChunks: []map[string]interface{}{{
+ "id": "page-1", "doc_id": "active-doc", "compile_kwd": "wiki_page",
+ "available_int": 0, "kc_payload": "page content", "slug_kwd": "entity/page",
+ "page_type_kwd": "entity", "title_kwd": "page", "source_doc_ids": []string{"active-doc"},
+ }}}
+ r := engineReader{eng: eng}
+ if _, err := r.LoadDocumentWikiPagesBySlugs(t.Context(), "tenant", "kb1", []string{"entity/page"}); err != nil {
+ t.Fatalf("load document Wiki pages: %v", err)
+ }
+ wikiFilter, ok := eng.lastSearchReq.Filter["doc_id"]
+ if !ok || !reflect.DeepEqual(wikiFilter, []string{"active-doc"}) {
+ t.Fatalf("document Wiki filter = %#v, want active document IDs", wikiFilter)
+ }
+
+ w := engineWriter{eng: eng}
+ if _, err := w.loadActiveDocumentWikiPages(t.Context(), "tenant", "kb1"); err != nil {
+ t.Fatalf("load active document Wiki pages: %v", err)
+ }
+ graphFilter, ok := eng.lastSearchReq.Filter["doc_id"]
+ if !ok || !reflect.DeepEqual(graphFilter, []string{"active-doc"}) {
+ t.Fatalf("graph Wiki filter = %#v, want active document IDs", graphFilter)
+ }
+}
+
// TestSearchSimilarFiltersByVariant asserts the B1/KNN contract: SearchSimilar
// scopes the engine query to available_int=1 AND compile_kwd=variant, and the
// in-memory dirty-row guard drops any row whose compile_kwd does not map back to
diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.go
index bd1dd2a5f5..586b73ef06 100644
--- a/internal/ingestion/knowledge_compile/structure_merge_test.go
+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go
@@ -11,10 +11,13 @@ import (
// fakeWriter is a Writer double that records the last WriteMergedStructure
// buckets so a test can assert the grouping behavior without an engine.
type fakeWriter struct {
- buckets []StructureBucket
+ buckets []StructureBucket
+ writeMergedCalls int
+ projectWikiGraphCalls int
}
func (f *fakeWriter) WriteMerged(context.Context, string, string, []kccommon.Product) error {
+ f.writeMergedCalls++
return nil
}
func (f *fakeWriter) WriteMergedStructure(_ context.Context, _, _ string, buckets []StructureBucket) error {
@@ -34,8 +37,11 @@ func (f *fakeWriter) DeleteDocLevelForDocs(context.Context, string, string, []st
func (f *fakeWriter) StripMergedSources(context.Context, string, string, []string) error {
return nil
}
-func (f *fakeWriter) ProjectWikiGraph(context.Context, string, string) error { return nil }
-func (f *fakeWriter) DropWikiGraph(context.Context, string, string) error { return nil }
+func (f *fakeWriter) ProjectWikiGraph(context.Context, string, string) error {
+ f.projectWikiGraphCalls++
+ return nil
+}
+func (f *fakeWriter) DropWikiGraph(context.Context, string, string) error { return nil }
// TestMergeStructureDataset_GroupsByName covers G1: structure products are
// bucketed by name, descriptions folded, source ids unioned, and only
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution_test.go b/internal/ingestion/knowledge_compile/wiki_contribution_test.go
index 5ba6d3143f..3a4c942991 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution_test.go
@@ -5,6 +5,7 @@ import (
"testing"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/service/nav"
)
func wikiContributionTestProduct(slug, content string, chunkIDs ...string) kccommon.Product {
@@ -66,6 +67,38 @@ func TestDiffWikiDocumentContributionTreatsDisabledDocumentAsRetraction(t *testi
}
}
+func TestProcessBatchRetractionWithoutWikiContributionSkipsMerge(t *testing.T) {
+ store := &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}
+ previousNav := nav.GetNavService()
+ nav.SetNavService(&recordingNavService{})
+ t.Cleanup(func() { nav.SetNavService(previousNav) })
+ fw := &fakeWriter{}
+ factoryCalled := false
+ consumer := NewConsumer(NewFakeScheduler(),
+ WithReader(&fakeReader{}),
+ WithWriter(fw),
+ withWikiContributionStore(store),
+ WithDeduperFactory(func(string) (Deduper, error) {
+ factoryCalled = true
+ return nil, nil
+ }),
+ )
+ if err := consumer.processBatch(context.Background(), "tenant-1", "kb-1", "", []BacklogEntry{{
+ DocID: "doc-1", EventType: string(EventTypeDisabled), Variants: []string{"wiki"},
+ }}, []string{kccommon.TaskTypeWiki}); err != nil {
+ t.Fatalf("processBatch failed: %v", err)
+ }
+ if factoryCalled {
+ t.Fatal("retraction-only Wiki event initialized the dataset merge path")
+ }
+ if fw.writeMergedCalls != 0 {
+ t.Fatalf("retraction-only Wiki event wrote %d merged product batches", fw.writeMergedCalls)
+ }
+ if fw.projectWikiGraphCalls != 0 {
+ t.Fatalf("retraction without a Wiki contribution projected graph %d times, want 0", fw.projectWikiGraphCalls)
+ }
+}
+
func TestProcessBatchSkipsUnchangedWikiOnlyEvent(t *testing.T) {
product := wikiContributionTestProduct("cao-cao", "same content", "chunk-1")
store := &memoryWikiContributionStore{items: map[string]wikiDocumentContribution{}}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index d7f1aa3084..b5a13744c6 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -1478,16 +1478,27 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, k
if eng == nil {
return nil, nil
}
+ enabledIDs, statusAvailable, err := enabledDocumentIDs(ctx, kb)
+ if err != nil {
+ return nil, fmt.Errorf("load enabled Wiki documents for graph: %w", err)
+ }
+ if statusAvailable && len(enabledIDs) == 0 {
+ return nil, nil
+ }
const batchSize = 2000
bySlug := make(map[string]wikiPageProjection)
for offset := 0; ; offset += batchSize {
+ filter := map[string]interface{}{
+ "compile_kwd": compileKwdWikiPage, "available_int": 0,
+ "scope_kwd": "doc", "kb_id": kb,
+ }
+ if statusAvailable {
+ filter["doc_id"] = enabledIDs
+ }
result, err := eng.Search(ctx, &types.SearchRequest{
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)},
KbIDs: []string{kb},
- Filter: map[string]interface{}{
- "compile_kwd": compileKwdWikiPage, "available_int": 0,
- "scope_kwd": "doc", "kb_id": kb,
- },
+ Filter: filter,
SelectFields: []string{
"slug_kwd", "page_type_kwd", "title_kwd", "entity_names_kwd",
"summary_with_weight", "outlinks_kwd", "source_doc_ids", "source_chunk_ids",
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index 120e7b614c..04c0dd8893 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -93,7 +93,10 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,
continue
}
}
- s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, docID)
+ // A status transition changes document availability, not its parsed
+ // content. The status event handles dataset-level retraction/re-enable;
+ // scheduling a delayed Wiki recompilation here creates duplicate events
+ // and a visible 20-second oscillation.
s.publishKnowledgeCompileStatusChange(ctx, kb.TenantID, doc.KbID, docID, statusInt)
result[docID] = map[string]string{"status": status}
}
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index 6c8eb0676b..002d1bf080 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -592,7 +592,10 @@ func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *ent
return err
}
}
- s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, doc.ID)
+ // A status transition changes document availability, not its parsed
+ // content. The status event handles dataset-level retraction/re-enable;
+ // scheduling a delayed Wiki recompilation here creates duplicate events
+ // and a visible 20-second oscillation.
s.publishKnowledgeCompileStatusChange(ctx, kb.TenantID, doc.KbID, doc.ID, status)
return nil
}
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index 13a520b243..3ee329dfca 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -2615,6 +2615,32 @@ func TestUpdateDocumentChunkAvailabilityTogglesFinalProducts(t *testing.T) {
}
}
+func TestDocumentKnowledgeCompileTypesIncludesUnavailableProducts(t *testing.T) {
+ docEngine := &sourceAvailabilityDocEngine{}
+ svc := testDocumentService(t)
+ svc.docEngine = docEngine
+
+ variants, _, err := svc.documentKnowledgeCompileTypes(t.Context(), "tenant-1", "kb-1", "doc-1")
+ if err != nil {
+ t.Fatalf("documentKnowledgeCompileTypes failed: %v", err)
+ }
+ if docEngine.search == nil || !docEngine.search.IncludeUnavailable {
+ t.Fatalf("knowledge compile type search = %#v, want unavailable products included", docEngine.search)
+ }
+ for _, want := range []string{"tree", "structure", "wiki"} {
+ found := false
+ for _, variant := range variants {
+ if variant == want {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Fatalf("variants = %v, missing %q", variants, want)
+ }
+ }
+}
+
func TestClearDocumentParseResultsIsIdempotentForStaleDocSnapshot(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.go
index d300088edc..cda55970ec 100644
--- a/internal/service/document/knowledge_compile_status.go
+++ b/internal/service/document/knowledge_compile_status.go
@@ -65,12 +65,13 @@ func (s *DocumentService) documentKnowledgeCompileTypes(ctx context.Context, ten
for offset := 0; ; offset += 1000 {
searchCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)
result, err := s.docEngine.Search(searchCtx, &types.SearchRequest{
- IndexNames: []string{indexName},
- KbIDs: []string{datasetID},
- Offset: offset,
- Limit: 1000,
- SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd"},
- Filter: map[string]any{"doc_id": []string{documentID}},
+ IndexNames: []string{indexName},
+ KbIDs: []string{datasetID},
+ Offset: offset,
+ Limit: 1000,
+ SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd"},
+ Filter: map[string]any{"doc_id": []string{documentID}},
+ IncludeUnavailable: true,
})
cancel()
if err != nil {