RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: skip empty knowledge compile events (#18728)
buua436e07b338c4dbc
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex c020f1200f..d7a95ad452 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -388,6 +388,12 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)// cannot leave the doc's dataset-level products unrebuilt.return fmt.Errorf("knowledge_compile: rebuild recover variants %s: %w", docID, rerr)}+ if len(variants) == 0 {+ // Documents without compiled products do not contribute to any+ // dataset-level variant and must not enqueue an empty completion+ // event that falls back to the legacy all-products path.+ continue+ }if perr := rs.Publish(ctx, tenant, kb, docID, string(EventTypeCompleted), variants); perr != nil {return fmt.Errorf("knowledge_compile: rebuild republish %s: %w", docID, perr)}diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex 10127128be..c506276754 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -356,6 +356,12 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wikif err := putWikiActiveStates(ctx, docEngine, activeStates); err != nil {return err}+ // Do not wake the dataset consumer when this document never had any Wiki+ // products and the dirty replacement produced none. A full replacement with+ // existing products still publishes so the consumer can retract them.+ if len(rows) == 0 && (existing == nil || len(existing.Chunks) == 0) {+ return nil+ }return knowledge_compile.PublishCompleted(ctx, request.TenantID, request.DatasetID, request.DocumentID, []string{"wiki"})}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 496b989018..358b4f3524 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -263,7 +263,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map// Ordinary source chunks stay available_int=1 (the index default).markCompiledProductsHidden(chunks)- oldCompiledProductIDs, err := s.loadDocumentCompiledProductIDs(ctx)+ oldCompiledProductIDs, oldCompiledVariants, err := s.loadDocumentCompiledState(ctx)if err != nil {return nil, err}@@ -288,13 +288,17 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map// and is best-effort / non-fatal — a delivery failure is logged but does not// fail the pipeline task.//- // The variants passed to PublishCompleted are the compile types this document- // produced, derived from the authoritative `compilation_template_kind_kwd` the- // KnowledgeCompiler component stamps on each compiled product (the resolved- // template's kind → KindToVariant, O2a whitelist). This is the compiler's- // runtime inference surfaced here, NOT a re-derivation from `compile_kwd`.- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, compiledVariants(chunks)); err != nil {- common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))+ // The variants passed to PublishCompleted are the union of the compile types+ // in the previous and current document generations. They are derived from the+ // authoritative `compilation_template_kind_kwd` the KnowledgeCompiler+ // component stamps on each compiled product (the resolved template's kind →+ // KindToVariant, O2a whitelist). Keeping the previous types lets the dataset+ // consumer retract stale merged products when a template is removed.+ eventVariants := mergeCompiledVariants(oldCompiledVariants, compiledVariants(chunks))+ if len(eventVariants) > 0 {+ if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, eventVariants); err != nil {+ common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))+ }}// Compilation products are derived artifacts and must not inflate the@@ -423,28 +427,29 @@ func markCompiledProductsHidden(chunks []map[string]any) {}}-// loadDocumentCompiledProductIDs snapshots the previous successful document+// loadDocumentCompiledState snapshots the previous successful document// compiler generation before the new pipeline output is written. Reading first// avoids relying on immediate search visibility after a bulk index write.-func (s *PipelineExecutor) loadDocumentCompiledProductIDs(ctx context.Context) ([]string, error) {+func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]string, []string, error) {docEngine := engine.Get()if docEngine == nil || s == nil || s.taskCtx == nil {- return nil, nil+ return nil, nil, nil}const pageSize = 1000indexName := fmt.Sprintf("ragflow_%s", s.taskCtx.Tenant.ID)oldIDs := make([]string, 0)+ oldProducts := make([]map[string]any, 0)for offset := 0; ; offset += pageSize {result, err := docEngine.Search(ctx, &enginetypes.SearchRequest{IndexNames: []string{indexName},KbIDs: []string{s.taskCtx.Doc.KbID},Offset: offset,Limit: pageSize,- SelectFields: []string{"id", "compile_kwd"},+ SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd"},Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},})if err != nil {- return nil, fmt.Errorf("load document compiler products: %w", err)+ return nil, nil, fmt.Errorf("load document compiler products: %w", err)}if result == nil || len(result.Chunks) == 0 {break@@ -453,6 +458,7 @@ func (s *PipelineExecutor) loadDocumentCompiledProductIDs(ctx context.Context) (if strings.TrimSpace(asCompiledKwd(row)) == "" {continue}+ oldProducts = append(oldProducts, row)if id := strings.TrimSpace(anyString(row["id"])); id != "" {oldIDs = append(oldIDs, id)}@@ -461,7 +467,7 @@ func (s *PipelineExecutor) loadDocumentCompiledProductIDs(ctx context.Context) (break}}- return oldIDs, nil+ return oldIDs, compiledVariants(oldProducts), nil}// reconcileDocumentCompiledProducts advances the document-level compiler@@ -550,6 +556,32 @@ func compiledVariants(chunks []map[string]any) []string {return out}+// mergeCompiledVariants returns the union of the previous and current+// document-level compiler variants. A reparse that removes a compiler template+// still needs to notify the dataset consumer so it can retract stale merged+// products; a document that never had compiler products produces an empty set+// and does not wake the dataset consumer at all.+func mergeCompiledVariants(previous, current []string) []string {+ seen := make(map[string]struct{}, len(previous)+len(current))+ for _, variants := range [][]string{previous, current} {+ for _, variant := range variants {+ variant = strings.TrimSpace(variant)+ if variant != "" {+ seen[variant] = struct{}{}+ }+ }+ }+ if len(seen) == 0 {+ return nil+ }+ merged := make([]string, 0, len(seen))+ for variant := range seen {+ merged = append(merged, variant)+ }+ sort.Strings(merged)+ return merged+}+// asCompiledKwd extracts the compile_kwd keyword value from a chunk map,// tolerating both a bare string and a list-wrapped keyword column from the// engine. It returns "" when absent.diff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex ab50c9a7e6..c0716f0333 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -3,6 +3,7 @@ package taskimport ("context""errors"+ "reflect""sync""testing""time"@@ -841,3 +842,15 @@ func TestCountOriginalChunkIDs(t *testing.T) {t.Fatalf("compiler products: got %d, want 2", n)}}++func TestMergeCompiledVariants(t *testing.T) {+ if got := mergeCompiledVariants(nil, nil); got != nil {+ t.Fatalf("empty variants = %v, want nil", got)+ }++ got := mergeCompiledVariants([]string{"wiki", "structure"}, []string{"wiki", "tree"})+ want := []string{"structure", "tree", "wiki"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("merged variants = %v, want %v", got, want)+ }+}