Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: scope document deletion compile events (#20108)

buua4360ecd01254dbc
diff --git a/internal/ingestion/knowledge_compile/dataset_log.go b/internal/ingestion/knowledge_compile/dataset_log.go
index a40a1c7865..fb02f2a270 100644
--- a/internal/ingestion/knowledge_compile/dataset_log.go
+++ b/internal/ingestion/knowledge_compile/dataset_log.go
@@ -106,6 +106,11 @@ func taskTypesForEntry(entry BacklogEntry) []string {
if taskTypes := taskTypesForVariants(entry.Variants); len(taskTypes) > 0 {
return taskTypes
}
+ if EventType(entry.EventType) == EventTypeDeleted || EventType(entry.EventType) == EventTypeDisabled {
+ // Retraction events must be scoped to the artifact types that existed on
+ // the document. Empty metadata means there is no safe type to log.
+ return nil
+ }
// An event with no routing metadata must remain visible in every possible
// dataset log. Returning the fallback per entry prevents an unknown entry
// from disappearing when other entries in the same batch are typed.
@@ -139,11 +144,13 @@ func taskTypesForEntries(entries []BacklogEntry) []string {
}
}
if len(seen) == 0 {
- // Legacy events did not carry routing metadata. A deletion can affect any
- // dataset artifact, so expose one log per supported category rather than
- // incorrectly labeling the event as Wiki.
- for _, taskType := range allDatasetTaskTypes {
- seen[taskType] = struct{}{}
+ for _, entry := range entries {
+ if EventType(entry.EventType) != EventTypeDeleted && EventType(entry.EventType) != EventTypeDisabled {
+ for _, taskType := range allDatasetTaskTypes {
+ seen[taskType] = struct{}{}
+ }
+ break
+ }
}
}
return sortedTaskTypes(seen)
@@ -171,8 +178,13 @@ func startDatasetCompileLog(ctx context.Context, tenantID, datasetID, claimToken
}
}
if len(groups) == 0 {
- for _, taskType := range allDatasetTaskTypes {
- groups[taskType] = entries
+ for _, entry := range entries {
+ if EventType(entry.EventType) != EventTypeDeleted && EventType(entry.EventType) != EventTypeDisabled {
+ for _, taskType := range allDatasetTaskTypes {
+ groups[taskType] = entries
+ }
+ break
+ }
}
}
return kcDB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
diff --git a/internal/ingestion/knowledge_compile/dataset_log_test.go b/internal/ingestion/knowledge_compile/dataset_log_test.go
index 14114e7718..8aa3559d5a 100644
--- a/internal/ingestion/knowledge_compile/dataset_log_test.go
+++ b/internal/ingestion/knowledge_compile/dataset_log_test.go
@@ -103,4 +103,8 @@ func TestTaskTypesForEntryLegacyFallbacks(t *testing.T) {
if !reflect.DeepEqual(got, allDatasetTaskTypes) {
t.Fatalf("unknown entry task types = %v, want %v", got, allDatasetTaskTypes)
}
+
+ if got = taskTypesForEntry(BacklogEntry{EventType: string(EventTypeDeleted)}); len(got) != 0 {
+ t.Fatalf("unscoped deletion task types = %v, want none", got)
+ }
}
diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.go
index 8102e9501e..fc6cf01a84 100644
--- a/internal/ingestion/knowledge_compile/event.go
+++ b/internal/ingestion/knowledge_compile/event.go
@@ -121,7 +121,10 @@ func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, va
// append + notify pairing). variants and taskTypes identify the document-level
// artifacts that were removed before the event was published.
func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, variants, taskTypes []string) error {
- if defaultPublisher == nil {
+ // A deletion event without artifact variants cannot be scoped safely. Do not
+ // enqueue it: the consumer treats empty routing metadata as a legacy
+ // all-variants event, which could rebuild unrelated dataset artifacts.
+ if defaultPublisher == nil || len(variants) == 0 {
return nil
}
return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), variants, taskTypes)
diff --git a/internal/ingestion/knowledge_compile/scheduler_test.go b/internal/ingestion/knowledge_compile/scheduler_test.go
index c70f07e36b..88d492a4ab 100644
--- a/internal/ingestion/knowledge_compile/scheduler_test.go
+++ b/internal/ingestion/knowledge_compile/scheduler_test.go
@@ -88,6 +88,24 @@ func TestPublishDeletedPreservesRemovedProductTypes(t *testing.T) {
}
}
+func TestPublishDeletedSkipsUnscopedEvent(t *testing.T) {
+ scheduler := NewFakeScheduler()
+ previousPublisher, previousClaimer := defaultPublisher, defaultClaimer
+ defaultPublisher, defaultClaimer = scheduler, scheduler
+ t.Cleanup(func() {
+ defaultPublisher, defaultClaimer = previousPublisher, previousClaimer
+ })
+
+ if err := PublishDeleted(t.Context(), "t1", "kb1", "d1", nil, nil); err != nil {
+ t.Fatalf("publish deleted: %v", err)
+ }
+ if _, ok, err := scheduler.Claim(t.Context(), "kb1"); err != nil {
+ t.Fatalf("claim deleted event: %v", err)
+ } else if ok {
+ t.Fatal("unscoped deletion should not enqueue an event")
+ }
+}
+
func TestFakeSchedulerProgressIsClaimScoped(t *testing.T) {
f := NewFakeScheduler()
if err := f.Publish(t.Context(), "t1", "kb1", "d1", string(EventTypeCompleted), nil, nil); err != nil {
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index f11463a6cb..10b0e76167 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -317,7 +317,7 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
}
variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)
if typeErr != nil {
- common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to resolve knowledge compile types for %s: %v", docID, typeErr))
+ return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)
}
ingestionTask, err := s.ingestionTaskDAO.GetByDocumentID(ctx, dao.DB, docID)
if err != nil {
@@ -340,6 +340,9 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
if err := s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {
return err
}
+ if len(variants) == 0 {
+ return nil
+ }
// File replacement/deletion uses this path instead of deleteDocumentFull.
// Publish the same deletion event so the dataset-level consumer removes the
// deleted document's contribution in both paths.
@@ -417,7 +420,7 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant
indexName := fmt.Sprintf("ragflow_%s", tenantID)
variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)
if typeErr != nil {
- common.Warn(fmt.Sprintf("deleteDocEngineData: failed to resolve knowledge compile types for %s: %v", docID, typeErr))
+ return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)
}
deleteCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)
_, delErr := s.docEngine.DeleteChunks(deleteCtx, map[string]interface{}{"doc_id": docID}, indexName, kbID)
@@ -426,6 +429,12 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant
common.Warn(fmt.Sprintf("deleteDocEngineData: failed to delete chunks for %s: %v", docID, delErr))
return fmt.Errorf("delete chunks for document %s: %w", docID, delErr)
}
+ if len(variants) == 0 {
+ if s.metadataSvc != nil {
+ _ = s.DeleteDocumentAllMetadata(ctx, docID) // logs internally
+ }
+ return nil
+ }
// Notify the dataset-level post-processing consumer (§11) that this document's
// source + per-doc compiled chunks are gone. The consumer removes the
// dataset-scoped merged products and triggers incremental re-dedup. Best-
diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.go
index 049ed5d884..d300088edc 100644
--- a/internal/service/document/knowledge_compile_status.go
+++ b/internal/service/document/knowledge_compile_status.go
@@ -38,12 +38,6 @@ func (s *DocumentService) publishKnowledgeCompileStatusChange(ctx context.Contex
if err != nil {
common.Warn("document mutation: failed to resolve knowledge compile variants",
zap.String("document_id", documentID), zap.Error(err))
- if status == 0 {
- if publishErr := knowledge_compile.PublishDisabled(ctx, tenantID, datasetID, documentID, nil, nil); publishErr != nil {
- common.Warn("document mutation: failed to publish fallback knowledge compile disable",
- zap.String("document_id", documentID), zap.Error(publishErr))
- }
- }
return
}
if len(variants) == 0 {