Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: skip empty knowledge compile events (#18728)

buua436e07b338c4dbc
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index c020f1200f..d7a95ad452 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -388,6 +388,12 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)
// cannot leave the doc's dataset-level products unrebuilt.
return fmt.Errorf("knowledge_compile: rebuild recover variants %s: %w", docID, rerr)
}
+ if len(variants) == 0 {
+ // Documents without compiled products do not contribute to any
+ // dataset-level variant and must not enqueue an empty completion
+ // event that falls back to the legacy all-products path.
+ continue
+ }
if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted), variants); perr != nil {
return fmt.Errorf("knowledge_compile: rebuild republish %s: %w", docID, perr)
}
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 10127128be..c506276754 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -356,6 +356,12 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
if err := putWikiActiveStates(ctx, docEngine, activeStates); err != nil {
return err
}
+ // Do not wake the dataset consumer when this document never had any Wiki
+ // products and the dirty replacement produced none. A full replacement with
+ // existing products still publishes so the consumer can retract them.
+ if len(rows) == 0 && (existing == nil || len(existing.Chunks) == 0) {
+ return nil
+ }
return knowledge_compile.PublishCompleted(ctx, request.TenantID, request.DatasetID, request.DocumentID, []string{"wiki"})
}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 496b989018..358b4f3524 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -263,7 +263,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// Ordinary source chunks stay available_int=1 (the index default).
markCompiledProductsHidden(chunks)
- oldCompiledProductIDs, err := s.loadDocumentCompiledProductIDs(ctx)
+ oldCompiledProductIDs, oldCompiledVariants, err := s.loadDocumentCompiledState(ctx)
if err != nil {
return nil, err
}
@@ -288,13 +288,17 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// and is best-effort / non-fatal — a delivery failure is logged but does not
// fail the pipeline task.
//
- // The variants passed to PublishCompleted are the compile types this document
- // produced, derived from the authoritative `compilation_template_kind_kwd` the
- // KnowledgeCompiler component stamps on each compiled product (the resolved
- // template's kind → KindToVariant, O2a whitelist). This is the compiler's
- // runtime inference surfaced here, NOT a re-derivation from `compile_kwd`.
- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, compiledVariants(chunks)); err != nil {
- common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))
+ // The variants passed to PublishCompleted are the union of the compile types
+ // in the previous and current document generations. They are derived from the
+ // authoritative `compilation_template_kind_kwd` the KnowledgeCompiler
+ // component stamps on each compiled product (the resolved template's kind →
+ // KindToVariant, O2a whitelist). Keeping the previous types lets the dataset
+ // consumer retract stale merged products when a template is removed.
+ eventVariants := mergeCompiledVariants(oldCompiledVariants, compiledVariants(chunks))
+ if len(eventVariants) > 0 {
+ if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, eventVariants); err != nil {
+ common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))
+ }
}
// Compilation products are derived artifacts and must not inflate the
@@ -423,28 +427,29 @@ func markCompiledProductsHidden(chunks []map[string]any) {
}
}
-// loadDocumentCompiledProductIDs snapshots the previous successful document
+// loadDocumentCompiledState snapshots the previous successful document
// compiler generation before the new pipeline output is written. Reading first
// avoids relying on immediate search visibility after a bulk index write.
-func (s *PipelineExecutor) loadDocumentCompiledProductIDs(ctx context.Context) ([]string, error) {
+func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, error) {
docEngine := engine.Get()
if docEngine == nil || s == nil || s.taskCtx == nil {
- return nil, nil
+ return nil, nil, nil
}
const pageSize = 1000
indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)
oldIDs := make([]string, 0)
+ oldProducts := make([]map[string]any, 0)
for offset := 0; ; offset += pageSize {
result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{
IndexNames: []string{indexName},
KbIDs: []string{s.taskCtx.Doc.KbID},
Offset: offset,
Limit: pageSize,
- SelectFields: []string{"id", "compile_kwd"},
+ SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd"},
Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},
})
if err != nil {
- return nil, fmt.Errorf("load document compiler products: %w", err)
+ return nil, nil, fmt.Errorf("load document compiler products: %w", err)
}
if result == nil || len(result.Chunks) == 0 {
break
@@ -453,6 +458,7 @@ func (s *PipelineExecutor) loadDocumentCompiledProductIDs(ctx context.Context) (
if strings.TrimSpace(asCompiledKwd(row)) == "" {
continue
}
+ oldProducts = append(oldProducts, row)
if id := strings.TrimSpace(anyString(row["id"])); id != "" {
oldIDs = append(oldIDs, id)
}
@@ -461,7 +467,7 @@ func (s *PipelineExecutor) loadDocumentCompiledProductIDs(ctx context.Context) (
break
}
}
- return oldIDs, nil
+ return oldIDs, compiledVariants(oldProducts), nil
}
// reconcileDocumentCompiledProducts advances the document-level compiler
@@ -550,6 +556,32 @@ func compiledVariants(chunks []map[string]any) []string {
return out
}
+// mergeCompiledVariants returns the union of the previous and current
+// document-level compiler variants. A reparse that removes a compiler template
+// still needs to notify the dataset consumer so it can retract stale merged
+// products; a document that never had compiler products produces an empty set
+// and does not wake the dataset consumer at all.
+func mergeCompiledVariants(previous, current []string) []string {
+ seen := make(map[string]struct{}, len(previous)+len(current))
+ for _, variants := range [][]string{previous, current} {
+ for _, variant := range variants {
+ variant = strings.TrimSpace(variant)
+ if variant != "" {
+ seen[variant] = struct{}{}
+ }
+ }
+ }
+ if len(seen) == 0 {
+ return nil
+ }
+ merged := make([]string, 0, len(seen))
+ for variant := range seen {
+ merged = append(merged, variant)
+ }
+ sort.Strings(merged)
+ return merged
+}
+
// asCompiledKwd extracts the compile_kwd keyword value from a chunk map,
// tolerating both a bare string and a list-wrapped keyword column from the
// engine. It returns "" when absent.
diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.go
index ab50c9a7e6..c0716f0333 100644
--- a/internal/ingestion/task/pipeline_executor_test.go
+++ b/internal/ingestion/task/pipeline_executor_test.go
@@ -3,6 +3,7 @@ package task
import (
"context"
"errors"
+ "reflect"
"sync"
"testing"
"time"
@@ -841,3 +842,15 @@ func TestCountOriginalChunkIDs(t *testing.T) {
t.Fatalf("compiler products: got %d, want 2", n)
}
}
+
+func TestMergeCompiledVariants(t *testing.T) {
+ if got := mergeCompiledVariants(nil, nil); got != nil {
+ t.Fatalf("empty variants = %v, want nil", got)
+ }
+
+ got := mergeCompiledVariants([]string{"wiki", "structure"}, []string{"wiki", "tree"})
+ want := []string{"structure", "tree", "wiki"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("merged variants = %v, want %v", got, want)
+ }
+}