RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(service): purge engine data from RemoveDocumentKeepFile (#20659) (#20768)
Harsh Kashyap730f12ba459a
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex 65b9b4efa6..e12965f467 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -1153,6 +1153,15 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfacdeleted = int64(d)}+ // delete_by_query can answer 200 while only part of the work was done.+ // Surface that so callers do not treat leftover chunks as deleted.+ if timedOut, _ := result["timed_out"].(bool); timedOut {+ return deleted, fmt.Errorf("elasticsearch delete by query timed out after deleting %d chunks from %s", deleted, fullIndexName)+ }+ if failures, ok := result["failures"].([]interface{}); ok && len(failures) > 0 {+ return deleted, fmt.Errorf("elasticsearch delete by query reported %d failures after deleting %d chunks from %s", len(failures), deleted, fullIndexName)+ }+common.Info("Successfully deleted chunks", zap.String("index_name", fullIndexName), zap.Int64("deleted_count", deleted))return deleted, nil}diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex ae5b3f89a2..d097303cb8 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -1014,6 +1014,78 @@ func TestMapMemoryMessageESUpdateFieldsRefreshesTokens(t *testing.T) {assertEqual(t, doc["status_int"], 0)}+// TestDeleteChunksSurfacesPartialFailures guards against delete_by_query+// answering 200 with timed_out or failures set while DeleteChunks reports nil.+func TestDeleteChunksSurfacesPartialFailures(t *testing.T) {+ cases := map[string]string{+ "timed_out": `{"deleted":2,"timed_out":true,"failures":[]}`,+ "failures": `{"deleted":2,"timed_out":false,"failures":[{"index":"ragflow_tenant","status":500}]}`,+ }+ for name, response := range cases {+ t.Run(name, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ if r.Method == http.MethodHead {+ w.WriteHeader(http.StatusOK)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(response))+ }))+ defer server.Close()+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ deleted, err := engine.DeleteChunks(t.Context(), map[string]interface{}{"kb_id": "kb-1", "doc_id": "doc-1"}, "ragflow_tenant", "kb-1")+ if err == nil {+ t.Fatal("DeleteChunks returned nil error for a partial delete_by_query response")+ }+ if deleted != 2 {+ t.Fatalf("deleted=%d, want 2", deleted)+ }+ })+ }+}++// TestDeleteMetadataSurfacesPartialFailures guards the metadata side of the+// same contract: delete_by_query answering 200 with timed_out or failures set+// must surface as an error, otherwise RemoveDocumentKeepFile can leave orphan+// ragflow_doc_meta_<tenant> rows (#20659).+func TestDeleteMetadataSurfacesPartialFailures(t *testing.T) {+ cases := map[string]string{+ "timed_out": `{"deleted":2,"timed_out":true,"failures":[]}`,+ "failures": `{"deleted":2,"timed_out":false,"failures":[{"index":"ragflow_doc_meta_tenant","status":500}]}`,+ }+ for name, response := range cases {+ t.Run(name, func(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ if r.Method == http.MethodHead {+ w.WriteHeader(http.StatusOK)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(response))+ }))+ defer server.Close()+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ deleted, err := engine.DeleteMetadata(t.Context(), map[string]interface{}{"kb_id": "kb-1", "id": "doc-1"}, "tenant")+ if err == nil {+ t.Fatal("DeleteMetadata returned nil error for a partial delete_by_query response")+ }+ if deleted != 2 {+ t.Fatalf("deleted=%d, want 2", deleted)+ }+ })+ }+}+func TestCloneFilterSnapshotIsolatesLaterShouldAppends(t *testing.T) {boolMap := map[string]interface{}{"must": []interface{}{map[string]interface{}{"term": map[string]interface{}{"kb_id": "kb-1"}}},diff --git a/internal/engine/elasticsearch/metadata.go b/internal/engine/elasticsearch/metadata.goindex bc7f8107cf..8c9aba6ccc 100644--- a/internal/engine/elasticsearch/metadata.go+++ b/internal/engine/elasticsearch/metadata.go@@ -319,6 +319,15 @@ func (e *Engine) DeleteMetadata(ctx context.Context, condition map[string]interfdeleted = int64(d)}+ // delete_by_query can answer 200 while only part of the work was done.+ // Surface that so callers do not treat leftover metadata as deleted.+ if timedOut, _ := result["timed_out"].(bool); timedOut {+ return deleted, fmt.Errorf("elasticsearch delete metadata by query timed out after deleting %d metadata documents from %s", deleted, indexName)+ }+ if failures, ok := result["failures"].([]interface{}); ok && len(failures) > 0 {+ return deleted, fmt.Errorf("elasticsearch delete metadata by query reported %d failures after deleting %d metadata documents from %s", len(failures), deleted, indexName)+ }+common.Info("ElasticsearchConnection.DeleteMetadata completes", zap.String("index_name", indexName), zap.Int64("deleted_count", deleted))return deleted, nil}diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex 53a165e6f5..3d56ad97bc 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -293,7 +293,11 @@ func (s *DocumentService) deleteDocumentFull(ctx context.Context, docID string)}}- if err := s.deleteDocEngineData(ctx, docID, kb.TenantID, doc.KbID); err != nil {+ variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)+ if typeErr != nil {+ return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)+ }+ if err := s.deleteDocEngineData(ctx, docID, kb.TenantID, doc.KbID, variants, taskTypes); err != nil {return err}if err = s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {@@ -313,6 +317,10 @@ func (s *DocumentService) deleteDocumentFull(ctx context.Context, docID string)// deleting the underlying file record, its storage blob, or its file2document// mappings. Mirrors Python DocumentService.remove_document — the caller is// responsible for cleaning up the file2document mappings separately.+//+// Chunk + metadata cleanup is delegated to deleteDocEngineData so the keep-+// file path matches deleteDocumentFull. The doc_deleted event is published+// from inside deleteDocEngineData; no separate PublishDeleted is needed here.func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID string) error {doc, kb, err := s.resolveDocAndKB(ctx, docID)if err != nil {@@ -337,19 +345,11 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri}common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to delete tasks for %s: %v", docID, delErr))}- if err := s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {+ if err := s.deleteDocEngineData(ctx, docID, kb.TenantID, doc.KbID, variants, taskTypes); err != nil {return err}- if len(variants) == 0 {- return nil- }- // File replacement/deletion uses this path instead of deleteDocumentFull.- // Publish the same deletion event so the dataset-level consumer removes the- // deleted document's contribution in both paths.- pubCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)- defer cancel()- if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID, variants, taskTypes); err != nil {- common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: publish doc_deleted for %s failed: %v", docID, err))+ if err := s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {+ return err}return nil}@@ -412,18 +412,22 @@ func (s *DocumentService) resolveDocAndKB(ctx context.Context, docID string) (*e}// deleteDocEngineData removes chunks and metadata from the document engine.-// No-op when the engine is nil.-func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantID, kbID string) error {+// No-op when the engine is nil. Callers pass the resolved compile variants and+// task types so the file-delete hot path does not search for them twice.+func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantID, kbID string, variants, taskTypes []string) error {if s.docEngine == nil {return nil}indexName := fmt.Sprintf("ragflow_%s", tenantID)- variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)- if typeErr != nil {- return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)- }deleteCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)_, delErr := s.docEngine.DeleteChunks(deleteCtx, map[string]interface{}{"doc_id": docID}, indexName, kbID)+ if delErr != nil && deleteCtx.Err() == nil {+ // Retry once under a fresh deadline: reusing deleteCtx would start the+ // second attempt with whatever time the first attempt left over.+ retryCtx, retryCancel := context.WithTimeout(ctx, cleanupBatchTimeout)+ _, delErr = s.docEngine.DeleteChunks(retryCtx, map[string]interface{}{"doc_id": docID}, indexName, kbID)+ retryCancel()+ }cancel()if delErr != nil {common.Warn(fmt.Sprintf("deleteDocEngineData: failed to delete chunks for %s: %v", docID, delErr))@@ -431,7 +435,9 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant}if len(variants) == 0 {if s.metadataSvc != nil {- _ = s.DeleteDocumentAllMetadata(ctx, docID) // logs internally+ if err := s.DeleteDocumentAllMetadata(ctx, docID); err != nil {+ return err+ }}return nil}@@ -448,7 +454,9 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantcommon.Warn(fmt.Sprintf("deleteDocEngineData: publish doc_deleted for %s failed: %v", docID, err))}if s.metadataSvc != nil {- _ = s.DeleteDocumentAllMetadata(ctx, docID) // logs internally+ if err := s.DeleteDocumentAllMetadata(ctx, docID); err != nil {+ return err+ }}return nil}diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.goindex a14633ab61..6b5cbdd121 100644--- a/internal/service/document/document_test.go+++ b/internal/service/document/document_test.go@@ -5440,6 +5440,246 @@ func TestFileDeleteRemovesLinkedDocument(t *testing.T) {}}+// keepFileCleanupDocEngine records DeleteChunks + DeleteMetadata calls so the+// keep-file cleanup path can assert that engine data is purged even when the+// underlying file record + storage blob are explicitly preserved.+type keepFileCleanupDocEngine struct {+ fakeChatDocEngine+ deleteChunksError error+ deleteChunksFailOnce bool+ deleteMetadataError error+ deleteChunksCalls int+ deleteChunksCondition map[string]interface{}+ deleteChunksIndexName string+ deleteChunksDatasetID string+ deleteMetadataCalls int+ deleteMetadataCond map[string]interface{}+}++func (e *keepFileCleanupDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {+ e.deleteChunksCalls+++ e.deleteChunksCondition = condition+ e.deleteChunksIndexName = indexName+ e.deleteChunksDatasetID = datasetID+ if e.deleteChunksFailOnce && e.deleteChunksCalls > 1 {+ return 1, nil+ }+ return 1, e.deleteChunksError+}++func (e *keepFileCleanupDocEngine) DeleteMetadata(_ context.Context, condition map[string]interface{}, _ string) (int64, error) {+ e.deleteMetadataCalls+++ e.deleteMetadataCond = condition+ return 0, e.deleteMetadataError+}++// TestRemoveDocumentKeepFileCleansUpEngineData regression-locks the+// cycle-108 fix: RemoveDocumentKeepFile must purge chunks + metadata from+// the engine, mirroring deleteDocumentFull. Pre-fix, the function left+// orphans in the chunk store and ragflow_doc_meta_<tenant> table, which+// MetadataService.GetFlattedMetaByKBs could still surface.+func TestRemoveDocumentKeepFileCleansUpEngineData(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")++ engine := &keepFileCleanupDocEngine{}+ svc := testDocumentService(t)+ svc.docEngine = engine+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)++ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err != nil {+ t.Fatalf("RemoveDocumentKeepFile failed: %v", err)+ }++ if engine.deleteChunksCalls != 1 {+ t.Fatalf("DeleteChunks calls = %d, want 1", engine.deleteChunksCalls)+ }+ if got := engine.deleteChunksIndexName; got != "ragflow_tenant-1" {+ t.Fatalf("DeleteChunks indexName = %q, want %q", got, "ragflow_tenant-1")+ }+ if got := engine.deleteChunksDatasetID; got != "kb-1" {+ t.Fatalf("DeleteChunks datasetID = %q, want %q", got, "kb-1")+ }+ if got := engine.deleteChunksCondition["doc_id"]; got != "doc-1" {+ t.Fatalf("DeleteChunks condition[doc_id] = %v, want %q", got, "doc-1")+ }++ if engine.deleteMetadataCalls != 1 {+ t.Fatalf("DeleteMetadata calls=%d, want 1", engine.deleteMetadataCalls)+ }+ if engine.deleteMetadataCond["id"] != "doc-1" || engine.deleteMetadataCond["kb_id"] != "kb-1" {+ t.Fatalf("metadata condition=%v", engine.deleteMetadataCond)+ }++ // Verify document row removed.+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err == nil {+ t.Fatal("document row should be removed")+ }++ // Verify ingestion task removed.+ if task, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1"); err != nil {+ t.Fatalf("reload ingestion task: %v", err)+ } else if task != nil {+ t.Fatal("ingestion task should be removed")+ }++ // Verify KB counters decremented.+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")+ if err != nil {+ t.Fatalf("reload kb: %v", err)+ }+ if kb.DocNum != 0 {+ t.Fatalf("doc_num = %d, want 0", kb.DocNum)+ }+}++// TestRemoveDocumentKeepFilePreservesFileRecord regression-locks the+// explicit "keep file" intent: the function must NOT touch the file row,+// the file2document mapping, or the storage blob. Pre-fix, the bug was+// about orphan data; this test pins the inverse: the file path stays+// untouched.+func TestRemoveDocumentKeepFilePreservesFileRecord(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")+ loc := "path/to/blob"+ insertTestFile(t, "file-1", "dataset-folder-1", "test.pdf", &loc)+ insertTestFile2Document(t, "f2d-1", "file-1", "doc-1")+ store := newFakeUploadStorage()+ if err := store.Put(t.Context(), "kb-1", loc, []byte("document")); err != nil {+ t.Fatalf("store document blob: %v", err)+ }+ factory := storage.GetStorageFactory()+ originalStorage := factory.GetStorage()+ factory.SetStorage(store)+ t.Cleanup(func() { factory.SetStorage(originalStorage) })++ engine := &keepFileCleanupDocEngine{}+ svc := testDocumentService(t)+ svc.docEngine = engine++ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err != nil {+ t.Fatalf("RemoveDocumentKeepFile failed: %v", err)+ }++ // File record should still exist.+ files, _ := dao.NewFileDAO().GetByIDs(t.Context(), db, []string{"file-1"})+ if len(files) != 1 {+ t.Fatalf("file record should survive, got %d", len(files))+ }+ // file2document mapping should still exist.+ mappings, _ := dao.NewFile2DocumentDAO().GetByDocumentID(t.Context(), db, "doc-1")+ if len(mappings) != 1 {+ t.Fatalf("file2document mapping should survive, got %d", len(mappings))+ }+ // Storage blob should still exist.+ if !store.ObjExist(t.Context(), "kb-1", loc) {+ t.Fatal("storage blob should survive")+ }+}++func TestRemoveDocumentKeepFileKeepsRowOnEngineFailure(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)+ svc := testDocumentService(t)+ svc.docEngine = &keepFileCleanupDocEngine{deleteChunksError: fmt.Errorf("partial delete")}+ svc.metadataSvc = nil+ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err == nil {+ t.Fatal("expected engine error")+ }+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err != nil {+ t.Fatalf("document row must remain: %v", err)+ }+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")+ if err != nil {+ t.Fatal(err)+ }+ if kb.DocNum != 1 {+ t.Fatalf("doc_num=%d, want 1", kb.DocNum)+ }+ if got := svc.docEngine.(*keepFileCleanupDocEngine).deleteChunksCalls; got != 2 {+ t.Fatalf("failed cleanup calls=%d, want bounded retry of 2", got)+ }+}++func TestRemoveDocumentKeepFileKeepsRowOnMetadataFailure(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)+ engine := &keepFileCleanupDocEngine{deleteMetadataError: fmt.Errorf("metadata unavailable")}+ svc := testDocumentService(t)+ svc.docEngine = engine+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)+ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err == nil {+ t.Fatal("expected metadata error")+ }+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err != nil {+ t.Fatalf("document row must remain: %v", err)+ }+ if engine.deleteMetadataCalls != 1 {+ t.Fatalf("metadata calls=%d", engine.deleteMetadataCalls)+ }+}++// TestDeleteDocumentFullKeepsDocumentWhenMetadataDeletionFails pins the+// shared-helper contract on the full path too: a metadata cleanup failure+// surfaces before the document row is removed, so a retry can complete the+// delete instead of leaving orphan ragflow_doc_meta_<tenant> records.+func TestDeleteDocumentFullKeepsDocumentWhenMetadataDeletionFails(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)+ engine := &keepFileCleanupDocEngine{deleteMetadataError: fmt.Errorf("metadata unavailable")}+ svc := testDocumentService(t)+ svc.docEngine = engine+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)+ if err := svc.deleteDocumentFull(t.Context(), "doc-1"); err == nil {+ t.Fatal("expected metadata error")+ }+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err != nil {+ t.Fatalf("document row must remain after metadata failure: %v", err)+ }+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")+ if err != nil {+ t.Fatal(err)+ }+ if kb.DocNum != 1 {+ t.Fatalf("doc_num=%d, want 1", kb.DocNum)+ }+ if engine.deleteMetadataCalls != 1 {+ t.Fatalf("metadata calls=%d, want 1", engine.deleteMetadataCalls)+ }+}++func TestRemoveDocumentKeepFileRetriesEngineFailure(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)+ engine := &keepFileCleanupDocEngine{deleteChunksError: fmt.Errorf("partial delete"), deleteChunksFailOnce: true}+ svc := testDocumentService(t)+ svc.docEngine = engine+ svc.metadataSvc = nil+ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err != nil {+ t.Fatalf("retry: %v", err)+ }+ if engine.deleteChunksCalls != 2 {+ t.Fatalf("calls=%d, want 2", engine.deleteChunksCalls)+ }+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err == nil {+ t.Fatal("document row should be removed after cleanup succeeds")+ }+}+func TestDownloadEmptyDocumentAndMissingBlob(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)