Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix delete dataset not delete bucket (#20110)

Wang Qi109dcdb0c24c
diff --git a/internal/admin/user_delete.go b/internal/admin/user_delete.go
index 36e68456fd..0796435da5 100644
--- a/internal/admin/user_delete.go
+++ b/internal/admin/user_delete.go
@@ -231,33 +231,34 @@ func (data *userDeletionData) deleteExternalData(ctx context.Context, docEngine
if document.Location == nil || *document.Location == "" {
continue
}
+ docNameID := namedDocument(document)
exists, err := store.ObjectExists(ctx, document.KbID, *document.Location)
if err != nil {
return fmt.Errorf("check document %s: %w", document.ID, err)
}
if !exists {
- common.Warn("Document object already missing", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
+ common.Warn("Document object already missing", zap.String("document", docNameID), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
continue
}
if err := store.Remove(ctx, document.KbID, *document.Location); err != nil {
- return fmt.Errorf("remove document %s: %w", document.ID, err)
+ return fmt.Errorf("remove document %s: %w", docNameID, err)
}
- common.Info("Removed document object", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
+ common.Info("Removed document object", zap.String("document", docNameID), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
}
for _, dataset := range data.datasets {
+ datasetNameID := namedID(dataset.Name, dataset.ID)
exists, err := store.BucketExistsWithError(ctx, dataset.ID)
if err != nil {
return fmt.Errorf("check dataset bucket %s: %w", dataset.ID, err)
}
if !exists {
- common.Warn("Dataset bucket already missing", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))
+ common.Warn("Dataset bucket already missing", zap.String("dataset", datasetNameID), zap.String("bucket", dataset.ID))
continue
}
- if err := store.RemoveEmptyBucket(ctx, dataset.ID); err != nil {
- common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID), zap.Error(err))
- } else {
- common.Info("Removed empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))
+ if err := store.RemoveBucket(ctx, dataset.ID); err != nil {
+ return fmt.Errorf("remove dataset bucket %s: %w", datasetNameID, err)
}
+ common.Info("Removed dataset bucket", zap.String("dataset", datasetNameID), zap.String("bucket", dataset.ID))
}
for _, file := range data.files {
if file.SourceType != string(entity.FileSourceKnowledgebase) && file.Location != nil && *file.Location != "" && file.Type != "folder" {
diff --git a/internal/admin/user_delete_test.go b/internal/admin/user_delete_test.go
index 410baf83b6..11e2a9abe9 100644
--- a/internal/admin/user_delete_test.go
+++ b/internal/admin/user_delete_test.go
@@ -72,17 +72,18 @@ func (e *deletionEngine) DeleteMetadata(_ context.Context, condition map[string]
type deletionStorage struct {
storage.Storage
- buckets []string
- files []string
- err error
- bucketErr error
- missing bool
- checkErr error
+ buckets []string
+ emptyBuckets []string
+ files []string
+ err error
+ bucketErr error
+ missing bool
+ checkErr error
}
func (s *deletionStorage) RemoveBucket(_ context.Context, bucket string) error {
- if s.err != nil {
- return s.err
+ if s.bucketErr != nil {
+ return s.bucketErr
}
s.buckets = append(s.buckets, bucket)
return nil
@@ -91,7 +92,7 @@ func (s *deletionStorage) RemoveEmptyBucket(_ context.Context, bucket string) er
if s.bucketErr != nil {
return s.bucketErr
}
- s.buckets = append(s.buckets, bucket)
+ s.emptyBuckets = append(s.emptyBuckets, bucket)
return nil
}
func (s *deletionStorage) Remove(_ context.Context, bucket, name string, _ ...string) error {
@@ -240,8 +241,8 @@ func TestDeleteUserRemovesOwnedDataAndJoinedDocuments(t *testing.T) {
if err := db.First(&team, "id = ?", "team-kb").Error; err != nil || team.DocNum != 1 || team.TokenNum != 3 || team.ChunkNum != 4 {
t.Fatalf("joined dataset counters = %+v, err=%v", team, err)
}
- if !slices.Equal(store.buckets, []string{"own-kb", "own-folder"}) || !slices.Equal(store.files, []string{"own-kb/own.pdf", "team-kb/joined.pdf", "own-folder/own.pdf", "team-folder/joined.pdf"}) {
- t.Fatalf("storage cleanup = buckets %v, files %v", store.buckets, store.files)
+ if !slices.Equal(store.buckets, []string{"own-kb"}) || !slices.Equal(store.emptyBuckets, []string{"own-folder"}) || !slices.Equal(store.files, []string{"own-kb/own.pdf", "team-kb/joined.pdf", "own-folder/own.pdf", "team-folder/joined.pdf"}) {
+ t.Fatalf("storage cleanup = buckets %v, empty buckets %v, files %v", store.buckets, store.emptyBuckets, store.files)
}
if !slices.Contains(docEngine.dropped, "ragflow_user-1") || !slices.Contains(docEngine.dropped, "memory_user-1") || !slices.Contains(docEngine.deleted, "doc_id:[joined-doc]") || !slices.Contains(docEngine.deleted, "metadata:id:[joined-doc]") {
t.Fatalf("index cleanup = dropped %v, deleted %v", docEngine.dropped, docEngine.deleted)
@@ -292,7 +293,7 @@ func TestDeleteUserKeepsUserWhenExternalCleanupFails(t *testing.T) {
}
}
-func TestDeleteUserContinuesWhenEmptyBucketRemovalFails(t *testing.T) {
+func TestDeleteUserKeepsUserWhenDatasetBucketRemovalFails(t *testing.T) {
db := setupUserDeletionDB(t)
if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
t.Fatal(err)
@@ -302,13 +303,13 @@ func TestDeleteUserContinuesWhenEmptyBucketRemovalFails(t *testing.T) {
}
service := NewService()
service.deleteEngine = &deletionEngine{}
- service.deleteStorage = &deletionStorage{bucketErr: errors.New("bucket is not empty")}
- if _, err := service.DeleteUser(t.Context(), "[email protected]"); err != nil {
- t.Fatalf("DeleteUser after bucket removal failure: %v", err)
+ service.deleteStorage = &deletionStorage{bucketErr: errors.New("storage unavailable")}
+ if _, err := service.DeleteUser(t.Context(), "[email protected]"); err == nil || !strings.Contains(err.Error(), "remove dataset bucket Dataset (dataset): storage unavailable") {
+ t.Fatalf("DeleteUser error = %v", err)
}
var count int64
- if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 0 {
- t.Fatalf("user retained after bucket failure: count=%d, err=%v", count, err)
+ if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 1 {
+ t.Fatalf("user deleted after bucket failure: count=%d, err=%v", count, err)
}
}
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index c53a88872f..21ea773d3b 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -337,7 +337,7 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
if storageImpl == nil {
return fmt.Errorf("storage not initialized")
}
- dataset := datasetNameAndID(kb)
+ datasetNameID := datasetNameAndID(kb)
// Collect document IDs first so engine cleanup can run before the
// transaction (engine ops are not transactional).
@@ -351,31 +351,31 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
if document.Location == nil || *document.Location == "" {
continue
}
+ docNameID := documentNameAndID(document)
exists, err := storageImpl.ObjectExists(cleanupCtx, kb.ID, *document.Location)
if err != nil {
return fmt.Errorf("check document %s in dataset %s: %w", document.ID, kb.ID, err)
}
if !exists {
- common.Warn("Dataset document object already missing", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))
+ common.Warn("Dataset document object already missing", zap.String("document", docNameID), zap.String("document_id", document.ID), zap.String("dataset", datasetNameID), zap.String("bucket", kb.ID))
continue
}
if err := storageImpl.Remove(cleanupCtx, kb.ID, *document.Location); err != nil {
- return fmt.Errorf("remove document %s from dataset %s: %w", document.ID, kb.ID, err)
+ return fmt.Errorf("remove document %s from dataset %s: %w", docNameID, kb.ID, err)
}
- common.Info("Removed dataset document object", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))
+ common.Info("Removed dataset document object", zap.String("document", docNameID), zap.String("document_id", document.ID), zap.String("dataset", datasetNameID), zap.String("bucket", kb.ID))
}
exists, err := storageImpl.BucketExistsWithError(cleanupCtx, kb.ID)
if err != nil {
return fmt.Errorf("check dataset bucket %s: %w", kb.ID, err)
}
if !exists {
- common.Warn("Dataset bucket already missing", zap.String("dataset", dataset), zap.String("bucket", kb.ID))
+ common.Warn("Dataset bucket already missing", zap.String("dataset", datasetNameID), zap.String("bucket", kb.ID))
} else {
- if err := storageImpl.RemoveEmptyBucket(cleanupCtx, kb.ID); err != nil {
- common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID), zap.Error(err))
- } else {
- common.Info("Removed empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID))
+ if err := storageImpl.RemoveBucket(cleanupCtx, kb.ID); err != nil {
+ return fmt.Errorf("remove dataset bucket for dataset %s: %w", datasetNameID, err)
}
+ common.Info("Removed dataset bucket", zap.String("dataset", datasetNameID), zap.String("bucket", kb.ID))
}
docIDs := extractDocIDs(documents)
if len(docIDs) > 0 {
@@ -428,7 +428,7 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
}); err != nil {
return err
}
- common.Info("Deleted dataset", zap.String("dataset", dataset), zap.String("kb_id", kb.ID))
+ common.Info("Deleted dataset", zap.String("dataset", datasetNameID), zap.String("kb_id", kb.ID))
return nil
}
diff --git a/internal/service/dataset/crud_delete_test.go b/internal/service/dataset/crud_delete_test.go
index cac67493b8..a2924f227a 100644
--- a/internal/service/dataset/crud_delete_test.go
+++ b/internal/service/dataset/crud_delete_test.go
@@ -29,7 +29,7 @@ import (
"go.uber.org/zap/zaptest/observer"
)
-func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {
+func TestDeleteDatasetRemovesTemporaryFiles(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)
insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Dataset")
@@ -40,7 +40,7 @@ func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {
if err := store.Put(t.Context(), "kb-1", "tracked", []byte("content")); err != nil {
t.Fatal(err)
}
- if err := store.Put(t.Context(), "kb-1", "untracked", []byte("content")); err != nil {
+ if err := store.Put(t.Context(), "kb-1", "temporary", []byte("content")); err != nil {
t.Fatal(err)
}
factory := storage.GetStorageFactory()
@@ -58,17 +58,20 @@ func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {
}
service := testDatasetUpdateService(t)
if err := service.deleteDataset(t.Context(), "tenant-1", kb); err != nil {
- t.Fatalf("delete dataset with nonempty bucket: %v", err)
+ t.Fatalf("delete dataset with temporary file: %v", err)
}
- if !store.ObjExist(t.Context(), "kb-1", "untracked") {
- t.Fatal("dataset deletion removed an untracked object")
+ if store.ObjExist(t.Context(), "kb-1", "temporary") {
+ t.Fatal("dataset deletion retained a temporary object")
}
if store.ObjExist(t.Context(), "kb-1", "tracked") {
t.Fatal("dataset deletion retained a document object")
}
+ if store.BucketExists(t.Context(), "kb-1") {
+ t.Fatal("dataset deletion retained the bucket")
+ }
var count int64
if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Count(&count).Error; err != nil || count != 0 {
- t.Fatalf("dataset retained after storage failure: count=%d, err=%v", count, err)
+ t.Fatalf("dataset retained after deletion: count=%d, err=%v", count, err)
}
if entries := logs.FilterMessage("Removed dataset document object").All(); len(entries) != 1 || entries[0].ContextMap()["document"] != "Document (doc-1)" || entries[0].ContextMap()["dataset"] != "Dataset (kb-1)" {
t.Errorf("document deletion logs = %v", entries)