RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix delete dataset not delete bucket (#20110)
Wang Qi109dcdb0c24c
diff --git a/internal/admin/user_delete.go b/internal/admin/user_delete.goindex 36e68456fd..0796435da5 100644--- a/internal/admin/user_delete.go+++ b/internal/admin/user_delete.go@@ -231,33 +231,34 @@ func (data *userDeletionData) deleteExternalData(ctx context.Context, docEngineif document.Location == nil || *document.Location == "" {continue}+ docNameID := namedDocument(document)exists, err := store.ObjectExists(ctx, document.KbID, *document.Location)if err != nil {return fmt.Errorf("check document %s: %w", document.ID, err)}if !exists {- common.Warn("Document object already missing", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))+ common.Warn("Document object already missing", zap.String("document", docNameID), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))continue}if err := store.Remove(ctx, document.KbID, *document.Location); err != nil {- return fmt.Errorf("remove document %s: %w", document.ID, err)+ return fmt.Errorf("remove document %s: %w", docNameID, err)}- common.Info("Removed document object", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))+ common.Info("Removed document object", zap.String("document", docNameID), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))}for _, dataset := range data.datasets {+ datasetNameID := namedID(dataset.Name, dataset.ID)exists, err := store.BucketExistsWithError(ctx, dataset.ID)if err != nil {return fmt.Errorf("check dataset bucket %s: %w", dataset.ID, err)}if !exists {- common.Warn("Dataset bucket already missing", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))+ common.Warn("Dataset bucket already missing", zap.String("dataset", datasetNameID), zap.String("bucket", dataset.ID))continue}- if err := store.RemoveEmptyBucket(ctx, dataset.ID); err != nil {- common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID), zap.Error(err))- } else {- common.Info("Removed empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))+ if err := store.RemoveBucket(ctx, dataset.ID); err != nil {+ return fmt.Errorf("remove dataset bucket %s: %w", datasetNameID, err)}+ common.Info("Removed dataset bucket", zap.String("dataset", datasetNameID), zap.String("bucket", dataset.ID))}for _, file := range data.files {if file.SourceType != string(entity.FileSourceKnowledgebase) && file.Location != nil && *file.Location != "" && file.Type != "folder" {diff --git a/internal/admin/user_delete_test.go b/internal/admin/user_delete_test.goindex 410baf83b6..11e2a9abe9 100644--- a/internal/admin/user_delete_test.go+++ b/internal/admin/user_delete_test.go@@ -72,17 +72,18 @@ func (e *deletionEngine) DeleteMetadata(_ context.Context, condition map[string]type deletionStorage struct {storage.Storage- buckets []string- files []string- err error- bucketErr error- missing bool- checkErr error+ buckets []string+ emptyBuckets []string+ files []string+ err error+ bucketErr error+ missing bool+ checkErr error}func (s *deletionStorage) RemoveBucket(_ context.Context, bucket string) error {- if s.err != nil {- return s.err+ if s.bucketErr != nil {+ return s.bucketErr}s.buckets = append(s.buckets, bucket)return nil@@ -91,7 +92,7 @@ func (s *deletionStorage) RemoveEmptyBucket(_ context.Context, bucket string) erif s.bucketErr != nil {return s.bucketErr}- s.buckets = append(s.buckets, bucket)+ s.emptyBuckets = append(s.emptyBuckets, bucket)return nil}func (s *deletionStorage) Remove(_ context.Context, bucket, name string, _ ...string) error {@@ -240,8 +241,8 @@ func TestDeleteUserRemovesOwnedDataAndJoinedDocuments(t *testing.T) {if err := db.First(&team, "id = ?", "team-kb").Error; err != nil || team.DocNum != 1 || team.TokenNum != 3 || team.ChunkNum != 4 {t.Fatalf("joined dataset counters = %+v, err=%v", team, err)}- if !slices.Equal(store.buckets, []string{"own-kb", "own-folder"}) || !slices.Equal(store.files, []string{"own-kb/own.pdf", "team-kb/joined.pdf", "own-folder/own.pdf", "team-folder/joined.pdf"}) {- t.Fatalf("storage cleanup = buckets %v, files %v", store.buckets, store.files)+ if !slices.Equal(store.buckets, []string{"own-kb"}) || !slices.Equal(store.emptyBuckets, []string{"own-folder"}) || !slices.Equal(store.files, []string{"own-kb/own.pdf", "team-kb/joined.pdf", "own-folder/own.pdf", "team-folder/joined.pdf"}) {+ t.Fatalf("storage cleanup = buckets %v, empty buckets %v, files %v", store.buckets, store.emptyBuckets, store.files)}if !slices.Contains(docEngine.dropped, "ragflow_user-1") || !slices.Contains(docEngine.dropped, "memory_user-1") || !slices.Contains(docEngine.deleted, "doc_id:[joined-doc]") || !slices.Contains(docEngine.deleted, "metadata:id:[joined-doc]") {t.Fatalf("index cleanup = dropped %v, deleted %v", docEngine.dropped, docEngine.deleted)@@ -292,7 +293,7 @@ func TestDeleteUserKeepsUserWhenExternalCleanupFails(t *testing.T) {}}-func TestDeleteUserContinuesWhenEmptyBucketRemovalFails(t *testing.T) {+func TestDeleteUserKeepsUserWhenDatasetBucketRemovalFails(t *testing.T) {db := setupUserDeletionDB(t)if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {t.Fatal(err)@@ -302,13 +303,13 @@ func TestDeleteUserContinuesWhenEmptyBucketRemovalFails(t *testing.T) {}service := NewService()service.deleteEngine = &deletionEngine{}- service.deleteStorage = &deletionStorage{bucketErr: errors.New("bucket is not empty")}- if _, err := service.DeleteUser(t.Context(), "[email protected]"); err != nil {- t.Fatalf("DeleteUser after bucket removal failure: %v", err)+ service.deleteStorage = &deletionStorage{bucketErr: errors.New("storage unavailable")}+ if _, err := service.DeleteUser(t.Context(), "[email protected]"); err == nil || !strings.Contains(err.Error(), "remove dataset bucket Dataset (dataset): storage unavailable") {+ t.Fatalf("DeleteUser error = %v", err)}var count int64- if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 0 {- t.Fatalf("user retained after bucket failure: count=%d, err=%v", count, err)+ if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 1 {+ t.Fatalf("user deleted after bucket failure: count=%d, err=%v", count, err)}}diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex c53a88872f..21ea773d3b 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -337,7 +337,7 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kbif storageImpl == nil {return fmt.Errorf("storage not initialized")}- dataset := datasetNameAndID(kb)+ datasetNameID := datasetNameAndID(kb)// Collect document IDs first so engine cleanup can run before the// transaction (engine ops are not transactional).@@ -351,31 +351,31 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kbif document.Location == nil || *document.Location == "" {continue}+ docNameID := documentNameAndID(document)exists, err := storageImpl.ObjectExists(cleanupCtx, kb.ID, *document.Location)if err != nil {return fmt.Errorf("check document %s in dataset %s: %w", document.ID, kb.ID, err)}if !exists {- common.Warn("Dataset document object already missing", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))+ common.Warn("Dataset document object already missing", zap.String("document", docNameID), zap.String("document_id", document.ID), zap.String("dataset", datasetNameID), zap.String("bucket", kb.ID))continue}if err := storageImpl.Remove(cleanupCtx, kb.ID, *document.Location); err != nil {- return fmt.Errorf("remove document %s from dataset %s: %w", document.ID, kb.ID, err)+ return fmt.Errorf("remove document %s from dataset %s: %w", docNameID, kb.ID, err)}- common.Info("Removed dataset document object", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))+ common.Info("Removed dataset document object", zap.String("document", docNameID), zap.String("document_id", document.ID), zap.String("dataset", datasetNameID), zap.String("bucket", kb.ID))}exists, err := storageImpl.BucketExistsWithError(cleanupCtx, kb.ID)if err != nil {return fmt.Errorf("check dataset bucket %s: %w", kb.ID, err)}if !exists {- common.Warn("Dataset bucket already missing", zap.String("dataset", dataset), zap.String("bucket", kb.ID))+ common.Warn("Dataset bucket already missing", zap.String("dataset", datasetNameID), zap.String("bucket", kb.ID))} else {- if err := storageImpl.RemoveEmptyBucket(cleanupCtx, kb.ID); err != nil {- common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID), zap.Error(err))- } else {- common.Info("Removed empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID))+ if err := storageImpl.RemoveBucket(cleanupCtx, kb.ID); err != nil {+ return fmt.Errorf("remove dataset bucket for dataset %s: %w", datasetNameID, err)}+ common.Info("Removed dataset bucket", zap.String("dataset", datasetNameID), zap.String("bucket", kb.ID))}docIDs := extractDocIDs(documents)if len(docIDs) > 0 {@@ -428,7 +428,7 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb}); err != nil {return err}- common.Info("Deleted dataset", zap.String("dataset", dataset), zap.String("kb_id", kb.ID))+ common.Info("Deleted dataset", zap.String("dataset", datasetNameID), zap.String("kb_id", kb.ID))return nil}diff --git a/internal/service/dataset/crud_delete_test.go b/internal/service/dataset/crud_delete_test.goindex cac67493b8..a2924f227a 100644--- a/internal/service/dataset/crud_delete_test.go+++ b/internal/service/dataset/crud_delete_test.go@@ -29,7 +29,7 @@ import ("go.uber.org/zap/zaptest/observer")-func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {+func TestDeleteDatasetRemovesTemporaryFiles(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Dataset")@@ -40,7 +40,7 @@ func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {if err := store.Put(t.Context(), "kb-1", "tracked", []byte("content")); err != nil {t.Fatal(err)}- if err := store.Put(t.Context(), "kb-1", "untracked", []byte("content")); err != nil {+ if err := store.Put(t.Context(), "kb-1", "temporary", []byte("content")); err != nil {t.Fatal(err)}factory := storage.GetStorageFactory()@@ -58,17 +58,20 @@ func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {}service := testDatasetUpdateService(t)if err := service.deleteDataset(t.Context(), "tenant-1", kb); err != nil {- t.Fatalf("delete dataset with nonempty bucket: %v", err)+ t.Fatalf("delete dataset with temporary file: %v", err)}- if !store.ObjExist(t.Context(), "kb-1", "untracked") {- t.Fatal("dataset deletion removed an untracked object")+ if store.ObjExist(t.Context(), "kb-1", "temporary") {+ t.Fatal("dataset deletion retained a temporary object")}if store.ObjExist(t.Context(), "kb-1", "tracked") {t.Fatal("dataset deletion retained a document object")}+ if store.BucketExists(t.Context(), "kb-1") {+ t.Fatal("dataset deletion retained the bucket")+ }var count int64if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Count(&count).Error; err != nil || count != 0 {- t.Fatalf("dataset retained after storage failure: count=%d, err=%v", count, err)+ t.Fatalf("dataset retained after deletion: count=%d, err=%v", count, err)}if entries := logs.FilterMessage("Removed dataset document object").All(); len(entries) != 1 || entries[0].ContextMap()["document"] != "Document (doc-1)" || entries[0].ContextMap()["dataset"] != "Dataset (kb-1)" {t.Errorf("document deletion logs = %v", entries)