RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: update knowledge artifacts on document status changes (#19532)
buua436c12224304210
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex d7a95ad452..f8cd8546d1 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -511,6 +511,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, etomb := c.tombs[kb]var completed []BacklogEntryvar deleted []string+ var disabled []BacklogEntry// Per-document tombstone clears are staged locally and committed only after// the write/delete paths below return without error, so a failed batch// leaves c.tombs untouched and can be retried on reclaim.@@ -553,6 +554,8 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e}tomb[docID] = 1deleted = append(deleted, docID)+ case EventTypeDisabled:+ disabled = append(disabled, BacklogEntry{DocID: docID, EventType: string(EventTypeDisabled), Variants: st.variants})case EventTypeCompleted:// A re-ingest after an earlier deletion: clear the prior// tombstone (deferred until the batch succeeds).@@ -562,6 +565,8 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e// Preserve the winning event's variants so the dispatch below can// route this doc to the matching dataset-level compile path (A0-4).completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted), Variants: st.variants})+ case EventTypeEnabled:+ completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeEnabled), Variants: st.variants})}}// Sort for deterministic iteration in the delete/load passes below.@@ -580,9 +585,10 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ezap.String("dataset_id", kb),zap.Int("completed_docs", len(completed)),zap.Int("deleted_docs", len(deleted)),+ zap.Int("disabled_docs", len(disabled)),zap.Strings("entries", entryDetails))- if len(deleted) == 0 && len(completed) == 0 {+ if len(deleted) == 0 && len(disabled) == 0 && len(completed) == 0 {c.reportProgress(ctx, tenant, kb, token, 1, "completed", "No document products require merging")return nil}@@ -613,54 +619,46 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ereturn nil}- deduper, err := c.factory(tenant)- if err != nil || deduper == nil {- // A no-op deduper would silently disable dataset-level LLM merging: every- // candidate (including e.g. a "吕布" wiki_page) would be written as its own- // merged row and duplicates would accumulate across runs. That degradation- // is never acceptable here, so fail loudly instead of papering over it.- common.Fatal("knowledge_compile: dataset-level LLM deduper unavailable, refusing to continue with a no-op merge",- zap.String("kb_id", kb),- zap.String("tenant_id", tenant),- zap.String("factory_err", func() string {- if err != nil {- return err.Error()- }- return "deduper factory returned nil"- }()))- }c.reportProgress(ctx, tenant, kb, token, 0.12, "deleting", fmt.Sprintf("Deleting products for %d document(s)", len(deleted)))- // --- Deletion (two sequential DocEngine calls, no in-memory load) ---- // Deleted wins regardless of batch order, so we process deletions first.- // The deleted docs' products are never loaded into memory: the DocEngine- // does all the work in two calls:+ // --- Retraction (sequential DocEngine calls, no in-memory load) ---+ // Deleted and disabled docs are handled before completion merges. Their+ // dataset-level contributions are removed without loading document products:+ // - deleted docs also lose their document-level compiled products;+ // - disabled docs retain document-level products for a later re-enable.+ // The DocEngine does the dataset-level work in two calls:// 1. DeleteDocLevelForDocs drops every per-document (doc-level) product of- // the deleted docs in a single engine call.+ // deleted docs in a single engine call.// 2. StripMergedSources removes the deleted doc ids from the source_doc_ids// array of every dataset-level merged product and deletes any product// whose array became empty.// A merged product referencing several deleted docs is pruned in one pass,// and an emptied product is removed exactly once.- if len(deleted) > 0 {- delIDs := make([]string, 0, len(deletedSet))+ if len(deleted) > 0 || len(disabled) > 0 {+ delIDs := make([]string, 0, len(deletedSet)+len(disabled))for d := range deletedSet {delIDs = append(delIDs, d)}+ for _, entry := range disabled {+ delIDs = append(delIDs, entry.DocID)+ }+ delIDs = uniqueSortedStrings(delIDs)// Each destructive write runs under the scheduler's per-dataset// write/rebuild lock with the generation check performed INSIDE the lock,// so a rewrite can neither land between the check and the write nor// interleave with the write itself (it must wait for this lock). This// closes the TOCTOU window where a worker past its fence could repopulate// storage the rebuild just cleared.- if err := c.withWriteLock(ctx, kb, token, func() error {- return c.writer.DeleteDocLevelForDocs(ctx, tenant, kb, delIDs)- }); err != nil {- if errors.Is(err, errClaimSuperseded) {- common.Info("knowledge_compile: batch stale before delete, aborting (rewrite barrier)",- zap.String("dataset_id", kb))+ if len(deleted) > 0 {+ if err := c.withWriteLock(ctx, kb, token, func() error {+ return c.writer.DeleteDocLevelForDocs(ctx, tenant, kb, deleted)+ }); err != nil {+ if errors.Is(err, errClaimSuperseded) {+ common.Info("knowledge_compile: batch stale before delete, aborting (rewrite barrier)",+ zap.String("dataset_id", kb))+ }+ return err}- return err}// The second destructive call is its own locked section: a rewrite can// land between the two, in which case the stale batch must not apply its@@ -686,6 +684,31 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, ereturn err}}+ if len(completed) == 0 && len(wikiDiff.affectedKeys) == 0 {+ c.mu.Lock()+ for _, docID := range pendingTombClear {+ delete(c.tombs[kb], docID)+ }+ c.mu.Unlock()+ c.reportProgress(ctx, tenant, kb, token, 1, "completed", "Document availability changes applied")+ return nil+ }+ deduper, err := c.factory(tenant)+ if err != nil || deduper == nil {+ // A no-op deduper would silently disable dataset-level LLM merging: every+ // candidate (including e.g. a "吕布" wiki_page) would be written as its own+ // merged row and duplicates would accumulate across runs. That degradation+ // is never acceptable here, so fail loudly instead of papering over it.+ common.Fatal("knowledge_compile: dataset-level LLM deduper unavailable, refusing to continue with a no-op merge",+ zap.String("kb_id", kb),+ zap.String("tenant_id", tenant),+ zap.String("factory_err", func() string {+ if err != nil {+ return err.Error()+ }+ return "deduper factory returned nil"+ }()))+ }// --- Completion merge ---// Load only the per-document products of the completed (and not deleted)diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.goindex fa949389af..bdc707b7f2 100644--- a/internal/ingestion/knowledge_compile/event.go+++ b/internal/ingestion/knowledge_compile/event.go@@ -44,10 +44,12 @@ type EventType stringconst (EventTypeCompleted EventType = "doc_completed"EventTypeDeleted EventType = "doc_deleted"+ EventTypeEnabled EventType = "doc_enabled"+ EventTypeDisabled EventType = "doc_disabled")-// KCCompileEvent is the payload published when a document's pipeline finishes-// (doc_completed) or is deleted (doc_deleted). It is intentionally decoupled+// KCCompileEvent is the payload published when a document's pipeline finishes,+// its availability changes, or it is deleted. It is intentionally decoupled// from common.TaskMessage because the consumer reads the raw JSON body.type KCCompileEvent struct {TenantID string `json:"tenant_id"`@@ -123,3 +125,22 @@ func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string) erro}return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), nil)}++// PublishEnabled records that a document with compiled products became+// available again and must be merged into dataset-level products.+func PublishEnabled(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {+ if defaultPublisher == nil {+ return nil+ }+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeEnabled), variants)+}++// PublishDisabled records that a document with compiled products became+// unavailable. The consumer retracts only its dataset-level contributions;+// document-level compiled rows remain available for a later re-enable.+func PublishDisabled(ctx context.Context, tenantID, datasetID, docID string, variants []string) error {+ if defaultPublisher == nil {+ return nil+ }+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDisabled), variants)+}diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex c123e74745..ea2929dbe9 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -94,6 +94,7 @@ func (s *DocumentService) BatchUpdateDocumentStatus(ctx context.Context, userID,}}s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, docID)+ s.publishKnowledgeCompileStatusChange(ctx, kb.TenantID, doc.KbID, docID, statusInt)result[docID] = map[string]string{"status": status}}diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex ce1c7f8da7..d4f4f826c8 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -570,5 +570,6 @@ func (s *DocumentService) updateDocumentStatusOnly(ctx context.Context, doc *ent}}s.markDocumentWikiDirty(ctx, kb.TenantID, doc.KbID, doc.ID)+ s.publishKnowledgeCompileStatusChange(ctx, kb.TenantID, doc.KbID, doc.ID, status)return nil}diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.gonew file mode 100644index 0000000000..cc0478b82a--- /dev/null+++ b/internal/service/document/knowledge_compile_status.go@@ -0,0 +1,99 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package document++import (+ "context"+ "fmt"+ "sort"+ "strings"++ "ragflow/internal/common"+ "ragflow/internal/engine/types"+ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+ knowledge_compile "ragflow/internal/ingestion/knowledge_compile"++ "go.uber.org/zap"+)++// publishKnowledgeCompileStatusChange notifies the dataset-level consumer when+// a document's retrieval availability changes. Documents without compiled+// products do not wake the consumer because they cannot contribute to any+// dataset-level knowledge artifact.+func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Context, tenantID, datasetID, documentID string, status int) {+ variants, err := s.documentKnowledgeCompileVariants(ctx, tenantID, datasetID, documentID)+ if err != nil {+ common.Warn("document mutation: failed to resolve knowledge compile variants",+ zap.String("document_id", documentID), zap.Error(err))+ return+ }+ if len(variants) == 0 {+ return+ }+ var publishErr error+ if status == 0 {+ publishErr = knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, variants)+ } else {+ publishErr = knowledge_compile.PublishEnabled(ctx, tenantID, datasetID, documentID, variants)+ }+ if publishErr != nil {+ common.Warn("document mutation: failed to publish knowledge compile status change",+ zap.String("document_id", documentID), zap.Int("status", status), zap.Error(publishErr))+ }+}++func (s *DocumentService) documentKnowledgeCompileVariants(ctx context.Context, tenantID, datasetID, documentID string) ([]string, error) {+ if s.docEngine == nil {+ return nil, nil+ }+ indexName := fmt.Sprintf("ragflow_%s", tenantID)+ seen := make(map[string]struct{})+ for offset := 0; ; offset += 1000 {+ result, err := s.docEngine.Search(ctx, &types.SearchRequest{+ IndexNames: []string{indexName},+ KbIDs: []string{datasetID},+ Offset: offset,+ Limit: 1000,+ SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd"},+ Filter: map[string]any{"doc_id": []string{documentID}},+ })+ if err != nil {+ return nil, err+ }+ if result == nil || len(result.Chunks) == 0 {+ break+ }+ for _, row := range result.Chunks {+ kind := strings.TrimSpace(documentStoreString(row["compilation_template_kind_kwd"]))+ variant, variantErr := kccommon.KindToVariant(kind)+ if variantErr != nil {+ variant, variantErr = knowledge_compile.KwdToVariant(strings.TrimSpace(documentStoreString(row["compile_kwd"])))+ }+ if variantErr == nil {+ seen[string(variant)] = struct{}{}+ }+ }+ if int64(offset+len(result.Chunks)) >= result.Total {+ break+ }+ }+ variants := make([]string, 0, len(seen))+ for variant := range seen {+ variants = append(variants, variant)+ }+ sort.Strings(variants)+ return variants, nil+}