RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: scope document deletion compile events (#20108)
buua4360ecd01254dbc
diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.goindex a40a1c7865..fb02f2a270 100644--- a/internal/ingestion/knowledge_compile/dataset_log.go+++ b/internal/ingestion/knowledge_compile/dataset_log.go@@ -106,6 +106,11 @@ func taskTypesForEntry(entry BacklogEntry) []string {if taskTypes := taskTypesForVariants(entry.Variants); len(taskTypes) > 0 {return taskTypes}+ if EventType(entry.EventType) == EventTypeDeleted || EventType(entry.EventType) == EventTypeDisabled {+ // Retraction events must be scoped to the artifact types that existed on+ // the document. Empty metadata means there is no safe type to log.+ return nil+ }// An event with no routing metadata must remain visible in every possible// dataset log. Returning the fallback per entry prevents an unknown entry// from disappearing when other entries in the same batch are typed.@@ -139,11 +144,13 @@ func taskTypesForEntries(entries []BacklogEntry) []string {}}if len(seen) == 0 {- // Legacy events did not carry routing metadata. A deletion can affect any- // dataset artifact, so expose one log per supported category rather than- // incorrectly labeling the event as Wiki.- for _, taskType := range allDatasetTaskTypes {- seen[taskType] = struct{}{}+ for _, entry := range entries {+ if EventType(entry.EventType) != EventTypeDeleted && EventType(entry.EventType) != EventTypeDisabled {+ for _, taskType := range allDatasetTaskTypes {+ seen[taskType] = struct{}{}+ }+ break+ }}}return sortedTaskTypes(seen)@@ -171,8 +178,13 @@ func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken}}if len(groups) == 0 {- for _, taskType := range allDatasetTaskTypes {- groups[taskType] = entries+ for _, entry := range entries {+ if EventType(entry.EventType) != EventTypeDeleted && EventType(entry.EventType) != EventTypeDisabled {+ for _, taskType := range allDatasetTaskTypes {+ groups[taskType] = entries+ }+ break+ }}}return kcDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.goindex 14114e7718..8aa3559d5a 100644--- a/internal/ingestion/knowledge_compile/dataset_log_test.go+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go@@ -103,4 +103,8 @@ func TestTaskTypesForEntryLegacyFallbacks(t *testing.T) {if !reflect.DeepEqual(got, allDatasetTaskTypes) {t.Fatalf("unknown entry task types = %v, want %v", got, allDatasetTaskTypes)}++ if got = taskTypesForEntry(BacklogEntry{EventType: string(EventTypeDeleted)}); len(got) != 0 {+ t.Fatalf("unscoped deletion task types = %v, want none", got)+ }}diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.goindex 8102e9501e..fc6cf01a84 100644--- a/internal/ingestion/knowledge_compile/event.go+++ b/internal/ingestion/knowledge_compile/event.go@@ -121,7 +121,10 @@ func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, va// append + notify pairing). variants and taskTypes identify the document-level// artifacts that were removed before the event was published.func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {- if defaultPublisher == nil {+ // A deletion event without artifact variants cannot be scoped safely. Do not+ // enqueue it: the consumer treats empty routing metadata as a legacy+ // all-variants event, which could rebuild unrelated dataset artifacts.+ if defaultPublisher == nil || len(variants) == 0 {return nil}return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), variants, taskTypes)diff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.goindex c70f07e36b..88d492a4ab 100644--- a/internal/ingestion/knowledge_compile/scheduler_test.go+++ b/internal/ingestion/knowledge_compile/scheduler_test.go@@ -88,6 +88,24 @@ func TestPublishDeletedPreservesRemovedProductTypes(t *testing.T) {}}+func TestPublishDeletedSkipsUnscopedEvent(t *testing.T) {+ scheduler := NewFakeScheduler()+ previousPublisher, previousClaimer := defaultPublisher, defaultClaimer+ defaultPublisher, defaultClaimer = scheduler, scheduler+ t.Cleanup(func() {+ defaultPublisher, defaultClaimer = previousPublisher, previousClaimer+ })++ if err := PublishDeleted(t.Context(), "t1", "kb1", "d1", nil, nil); err != nil {+ t.Fatalf("publish deleted: %v", err)+ }+ if _, ok, err := scheduler.Claim(t.Context(), "kb1"); err != nil {+ t.Fatalf("claim deleted event: %v", err)+ } else if ok {+ t.Fatal("unscoped deletion should not enqueue an event")+ }+}+func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {f := NewFakeScheduler()if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil, nil); err != nil {diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex f11463a6cb..10b0e76167 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -317,7 +317,7 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri}variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)if typeErr != nil {- common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to resolve knowledge compile types for %s: %v", docID, typeErr))+ return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)}ingestionTask, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, docID)if err != nil {@@ -340,6 +340,9 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID striif err := s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {return err}+ if len(variants) == 0 {+ return nil+ }// File replacement/deletion uses this path instead of deleteDocumentFull.// Publish the same deletion event so the dataset-level consumer removes the// deleted document's contribution in both paths.@@ -417,7 +420,7 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantindexName := fmt.Sprintf("ragflow_%s", tenantID)variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)if typeErr != nil {- common.Warn(fmt.Sprintf("deleteDocEngineData: failed to resolve knowledge compile types for %s: %v", docID, typeErr))+ return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)}deleteCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)_, delErr := s.docEngine.DeleteChunks(deleteCtx, map[string]interface{}{"doc_id": docID}, indexName, kbID)@@ -426,6 +429,12 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantcommon.Warn(fmt.Sprintf("deleteDocEngineData: failed to delete chunks for %s: %v", docID, delErr))return fmt.Errorf("delete chunks for document %s: %w", docID, delErr)}+ if len(variants) == 0 {+ if s.metadataSvc != nil {+ _ = s.DeleteDocumentAllMetadata(ctx, docID) // logs internally+ }+ return nil+ }// Notify the dataset-level post-processing consumer (§11) that this document's// source + per-doc compiled chunks are gone. The consumer removes the// dataset-scoped merged products and triggers incremental re-dedup. Best-diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.goindex 049ed5d884..d300088edc 100644--- a/internal/service/document/knowledge_compile_status.go+++ b/internal/service/document/knowledge_compile_status.go@@ -38,12 +38,6 @@ func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Contexif err != nil {common.Warn("document mutation: failed to resolve knowledge compile variants",zap.String("document_id", documentID), zap.Error(err))- if status == 0 {- if publishErr := knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, nil, nil); publishErr != nil {- common.Warn("document mutation: failed to publish fallback knowledge compile disable",- zap.String("document_id", documentID), zap.Error(publishErr))- }- }return}if len(variants) == 0 {