Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: update knowledge artifacts on document status changes (#19532)

buua436c12224304210
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index d7a95ad452..f8cd8546d1 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -511,6 +511,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
tomb := c.tombs[kb]
var completed []BacklogEntry
var deleted []string
+ var disabled []BacklogEntry
// Per-document tombstone clears are staged locally and committed only after
// the write/delete paths below return without error, so a failed batch
// leaves c.tombs untouched and can be retried on reclaim.
@@ -553,6 +554,8 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
}
tomb[docID] = 1
deleted = append(deleted, docID)
+ case EventTypeDisabled:
+ disabled = append(disabled, BacklogEntry{DocID: docID, EventType: string(EventTypeDisabled), Variants: st.variants})
case EventTypeCompleted:
// A re-ingest after an earlier deletion: clear the prior
// tombstone (deferred until the batch succeeds).
@@ -562,6 +565,8 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
// Preserve the winning event's variants so the dispatch below can
// route this doc to the matching dataset-level compile path (A0-4).
completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted), Variants: st.variants})
+ case EventTypeEnabled:
+ completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeEnabled), Variants: st.variants})
}
}
// Sort for deterministic iteration in the delete/load passes below.
@@ -580,9 +585,10 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
zap.String("dataset_id", kb),
zap.Int("completed_docs", len(completed)),
zap.Int("deleted_docs", len(deleted)),
+ zap.Int("disabled_docs", len(disabled)),
zap.Strings("entries", entryDetails))
- if len(deleted) == 0 && len(completed) == 0 {
+ if len(deleted) == 0 && len(disabled) == 0 && len(completed) == 0 {
c.reportProgress(ctx, tenant, kb, token, 1, "completed", "No document products require merging")
return nil
}
@@ -613,54 +619,46 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
return nil
}
- deduper, err := c.factory(tenant)
- if err != nil || deduper == nil {
- // A no-op deduper would silently disable dataset-level LLM merging: every
- // candidate (including e.g. a "吕布" wiki_page) would be written as its own
- // merged row and duplicates would accumulate across runs. That degradation
- // is never acceptable here, so fail loudly instead of papering over it.
- common.Fatal("knowledge_compile: dataset-level LLM deduper unavailable, refusing to continue with a no-op merge",
- zap.String("kb_id", kb),
- zap.String("tenant_id", tenant),
- zap.String("factory_err", func() string {
- if err != nil {
- return err.Error()
- }
- return "deduper factory returned nil"
- }()))
- }
c.reportProgress(ctx, tenant, kb, token, 0.12, "deleting", fmt.Sprintf("Deleting products for %d document(s)", len(deleted)))
- // --- Deletion (two sequential DocEngine calls, no in-memory load) ---
- // Deleted wins regardless of batch order, so we process deletions first.
- // The deleted docs' products are never loaded into memory: the DocEngine
- // does all the work in two calls:
+ // --- Retraction (sequential DocEngine calls, no in-memory load) ---
+ // Deleted and disabled docs are handled before completion merges. Their
+ // dataset-level contributions are removed without loading document products:
+ // - deleted docs also lose their document-level compiled products;
+ // - disabled docs retain document-level products for a later re-enable.
+ // The DocEngine does the dataset-level work in two calls:
// 1. DeleteDocLevelForDocs drops every per-document (doc-level) product of
- // the deleted docs in a single engine call.
+ // deleted docs in a single engine call.
// 2. StripMergedSources removes the deleted doc ids from the source_doc_ids
// array of every dataset-level merged product and deletes any product
// whose array became empty.
// A merged product referencing several deleted docs is pruned in one pass,
// and an emptied product is removed exactly once.
- if len(deleted) > 0 {
- delIDs := make([]string, 0, len(deletedSet))
+ if len(deleted) > 0 || len(disabled) > 0 {
+ delIDs := make([]string, 0, len(deletedSet)+len(disabled))
for d := range deletedSet {
delIDs = append(delIDs, d)
}
+ for _, entry := range disabled {
+ delIDs = append(delIDs, entry.DocID)
+ }
+ delIDs = uniqueSortedStrings(delIDs)
// Each destructive write runs under the scheduler's per-dataset
// write/rebuild lock with the generation check performed INSIDE the lock,
// so a rewrite can neither land between the check and the write nor
// interleave with the write itself (it must wait for this lock). This
// closes the TOCTOU window where a worker past its fence could repopulate
// storage the rebuild just cleared.
- if err := c.withWriteLock(ctx, kb, token, func() error {
- return c.writer.DeleteDocLevelForDocs(ctx, tenant, kb, delIDs)
- }); err != nil {
- if errors.Is(err, errClaimSuperseded) {
- common.Info("knowledge_compile: batch stale before delete, aborting (rewrite barrier)",
- zap.String("dataset_id", kb))
+ if len(deleted) > 0 {
+ if err := c.withWriteLock(ctx, kb, token, func() error {
+ return c.writer.DeleteDocLevelForDocs(ctx, tenant, kb, deleted)
+ }); err != nil {
+ if errors.Is(err, errClaimSuperseded) {
+ common.Info("knowledge_compile: batch stale before delete, aborting (rewrite barrier)",
+ zap.String("dataset_id", kb))
+ }
+ return err
}
- return err
}
// The second destructive call is its own locked section: a rewrite can
// land between the two, in which case the stale batch must not apply its
@@ -686,6 +684,31 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
return err
}
}
+ if len(completed) == 0 && len(wikiDiff.affectedKeys) == 0 {
+ c.mu.Lock()
+ for _, docID := range pendingTombClear {
+ delete(c.tombs[kb], docID)
+ }
+ c.mu.Unlock()
+ c.reportProgress(ctx, tenant, kb, token, 1, "completed", "Document availability changes applied")
+ return nil
+ }
+ deduper, err := c.factory(tenant)
+ if err != nil || deduper == nil {
+ // A no-op deduper would silently disable dataset-level LLM merging: every
+ // candidate (including e.g. a "吕布" wiki_page) would be written as its own
+ // merged row and duplicates would accumulate across runs. That degradation
+ // is never acceptable here, so fail loudly instead of papering over it.
+ common.Fatal("knowledge_compile: dataset-level LLM deduper unavailable, refusing to continue with a no-op merge",
+ zap.String("kb_id", kb),
+ zap.String("tenant_id", tenant),
+ zap.String("factory_err", func() string {
+ if err != nil {
+ return err.Error()
+ }
+ return "deduper factory returned nil"
+ }()))
+ }
// --- Completion merge ---
// Load only the per-document products of the completed (and not deleted)
diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.go
index fa949389af..bdc707b7f2 100644
--- a/internal/ingestion/knowledge_compile/event.go
+++ b/internal/ingestion/knowledge_compile/event.go
@@ -44,10 +44,12 @@ type EventType string
const (
EventTypeCompleted EventType = "doc_completed"
EventTypeDeleted EventType = "doc_deleted"
+ EventTypeEnabled EventType = "doc_enabled"
+ EventTypeDisabled EventType = "doc_disabled"
)
-// KCCompileEvent is the payload published when a document's pipeline finishes
-// (doc_completed) or is deleted (doc_deleted). It is intentionally decoupled
+// KCCompileEvent is the payload published when a document's pipeline finishes,
+// its availability changes, or it is deleted. It is intentionally decoupled
// from common.TaskMessage because the consumer reads the raw JSON body.
type KCCompileEvent struct {
TenantID string `json:"tenant_id"`
@@ -123,3 +125,22 @@ func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string) erro
}
return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil)
}
+
+// PublishEnabled records that a document with compiled products became
+// available again and must be merged into dataset-level products.
+func PublishEnabled(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {
+ if defaultPublisher == nil {
+ return nil
+ }
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeEnabled), variants)
+}
+
+// PublishDisabled records that a document with compiled products became
+// unavailable. The consumer retracts only its dataset-level contributions;
+// document-level compiled rows remain available for a later re-enable.
+func PublishDisabled(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {
+ if defaultPublisher == nil {
+ return nil
+ }
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDisabled), variants)
+}
diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.go
index c123e74745..ea2929dbe9 100644
--- a/internal/service/document/document_dataset_update.go
+++ b/internal/service/document/document_dataset_update.go
@@ -94,6 +94,7 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,
}
}
s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, docID)
+ s.publishKnowledgeCompileStatusChange(ctx, kb.TenantID, doc.KbID, docID, statusInt)
result[docID] = map[string]string{"status": status}
}
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index ce1c7f8da7..d4f4f826c8 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -570,5 +570,6 @@ func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *ent
}
}
s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, doc.ID)
+ s.publishKnowledgeCompileStatusChange(ctx, kb.TenantID, doc.KbID, doc.ID, status)
return nil
}
diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.go
new file mode 100644
index 0000000000..cc0478b82a
--- /dev/null
+++ b/internal/service/document/knowledge_compile_status.go
@@ -0,0 +1,99 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+ "context"
+ "fmt"
+ "sort"
+ "strings"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine/types"
+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ knowledge_compile "ragflow/internal/ingestion/knowledge_compile"
+
+ "go.uber.org/zap"
+)
+
+// publishKnowledgeCompileStatusChange notifies the dataset-level consumer when
+// a document's retrieval availability changes. Documents without compiled
+// products do not wake the consumer because they cannot contribute to any
+// dataset-level knowledge artifact.
+func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Context, tenantID, datasetID, documentID string, status int) {
+ variants, err := s.documentKnowledgeCompileVariants(ctx, tenantID, datasetID, documentID)
+ if err != nil {
+ common.Warn("document mutation: failed to resolve knowledge compile variants",
+ zap.String("document_id", documentID), zap.Error(err))
+ return
+ }
+ if len(variants) == 0 {
+ return
+ }
+ var publishErr error
+ if status == 0 {
+ publishErr = knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, variants)
+ } else {
+ publishErr = knowledge_compile.PublishEnabled(ctx, tenantID, datasetID, documentID, variants)
+ }
+ if publishErr != nil {
+ common.Warn("document mutation: failed to publish knowledge compile status change",
+ zap.String("document_id", documentID), zap.Int("status", status), zap.Error(publishErr))
+ }
+}
+
+func (s *DocumentService) documentKnowledgeCompileVariants(ctx context.Context, tenantID, datasetID, documentID string) ([]string, error) {
+ if s.docEngine == nil {
+ return nil, nil
+ }
+ indexName := fmt.Sprintf("ragflow_%s", tenantID)
+ seen := make(map[string]struct{})
+ for offset := 0; ; offset += 1000 {
+ result, err := s.docEngine.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{indexName},
+ KbIDs: []string{datasetID},
+ Offset: offset,
+ Limit: 1000,
+ SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd"},
+ Filter: map[string]any{"doc_id": []string{documentID}},
+ })
+ if err != nil {
+ return nil, err
+ }
+ if result == nil || len(result.Chunks) == 0 {
+ break
+ }
+ for _, row := range result.Chunks {
+ kind := strings.TrimSpace(documentStoreString(row["compilation_template_kind_kwd"]))
+ variant, variantErr := kccommon.KindToVariant(kind)
+ if variantErr != nil {
+ variant, variantErr = knowledge_compile.KwdToVariant(strings.TrimSpace(documentStoreString(row["compile_kwd"])))
+ }
+ if variantErr == nil {
+ seen[string(variant)] = struct{}{}
+ }
+ }
+ if int64(offset+len(result.Chunks)) >= result.Total {
+ break
+ }
+ }
+ variants := make([]string, 0, len(seen))
+ for variant := range seen {
+ variants = append(variants, variant)
+ }
+ sort.Strings(variants)
+ return variants, nil
+}