Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: delete datasets does not delete bucket and files in minio (#20024)

Wang Qif16e16df1671
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index 81c66434a7..72c9a46690 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -5,12 +5,14 @@ import (
"errors"
"fmt"
"strings"
+ "time"
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
pipelinepkg "ragflow/internal/ingestion/pipeline"
"ragflow/internal/service"
+ "ragflow/internal/storage"
"ragflow/internal/utility"
"go.uber.org/zap"
@@ -333,6 +335,11 @@ func (d *DatasetService) DeleteDatasets(ctx context.Context, ids []string, delet
}
func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb *entity.Knowledgebase) error {
+ storageImpl := storage.GetStorageFactory().GetStorage()
+ if storageImpl == nil {
+ return fmt.Errorf("storage not initialized")
+ }
+
// Collect document IDs first so engine cleanup can run before the
// transaction (engine ops are not transactional).
var documents []entity.Document
@@ -344,7 +351,7 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
d.deleteDatasetEngineData(ctx, kb, docIDs)
}
- return dao.DB.Transaction(func(tx *gorm.DB) error {
+ if err := dao.DB.Transaction(func(tx *gorm.DB) error {
// Delete index tasks referencing this KB.
if taskIDs := datasetIndexTaskIDs(kb); len(taskIDs) > 0 {
if err := tx.Where("id IN ?", taskIDs).Delete(&entity.Task{}).Error; err != nil {
@@ -387,7 +394,16 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
return fmt.Errorf("delete dataset error for %s", kb.ID)
}
return nil
- })
+ }); err != nil {
+ return err
+ }
+ // Dataset uploads and generated objects are stored under the KB ID.
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute)
+ defer cancel()
+ if err := storageImpl.RemoveBucket(cleanupCtx, kb.ID); err != nil {
+ return fmt.Errorf("dataset %s deleted but storage cleanup failed: %w", kb.ID, err)
+ }
+ return nil
}
func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, page, pageSize int, terms []dao.OrderTerm, keywords string, ownerIDs []string, parserID, userID string, ids []string) ([]map[string]interface{}, int64, common.ErrorCode, error) {
diff --git a/internal/storage/minio.go b/internal/storage/minio.go
index 141d170b60..df5342c6d5 100644
--- a/internal/storage/minio.go
+++ b/internal/storage/minio.go
@@ -328,6 +328,13 @@ func (m *MinioStorage) RemoveBucket(ctx context.Context, bucket string) error {
if m.bucket != "" {
actualBucket = m.bucket
}
+ exists, err := m.client.BucketExists(ctx, actualBucket)
+ if err != nil {
+ return fmt.Errorf("failed to check bucket %s: %w", actualBucket, err)
+ }
+ if !exists {
+ return nil
+ }
// Build prefix for single-bucket mode
prefix := ""
@@ -338,25 +345,49 @@ func (m *MinioStorage) RemoveBucket(ctx context.Context, bucket string) error {
prefix += fmt.Sprintf("%s/", origBucket)
}
- // List and delete objects with prefix
+ // Include versions and delete markers so versioned buckets can be emptied.
+ removeCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
objectsCh := make(chan minio.ObjectInfo)
+ listErrCh := make(chan error, 1)
go func() {
defer close(objectsCh)
- for obj := range m.client.ListObjects(ctx, actualBucket, minio.ListObjectsOptions{
- Prefix: prefix,
- Recursive: true,
+ defer close(listErrCh)
+ for obj := range m.client.ListObjects(removeCtx, actualBucket, minio.ListObjectsOptions{
+ Prefix: prefix,
+ Recursive: true,
+ WithVersions: true,
}) {
if obj.Err != nil {
common.Warn("Failed to list objects", zap.Error(obj.Err))
+ listErrCh <- obj.Err
+ return
+ }
+ select {
+ case objectsCh <- obj:
+ case <-removeCtx.Done():
return
}
- objectsCh <- obj
}
}()
- for err := range m.client.RemoveObjects(ctx, actualBucket, objectsCh, minio.RemoveObjectsOptions{}) {
- common.Warn(fmt.Sprintf("Failed to remove object, key: %s", err.ObjectName), zap.Error(err.Err))
+ var removeErr error
+ for objErr := range m.client.RemoveObjects(removeCtx, actualBucket, objectsCh, minio.RemoveObjectsOptions{}) {
+ common.Warn(fmt.Sprintf("Failed to remove object, key: %s", objErr.ObjectName), zap.Error(objErr.Err))
+ if removeErr == nil {
+ removeErr = fmt.Errorf("failed to remove object %s: %w", objErr.ObjectName, objErr.Err)
+ }
+ }
+ cancel()
+ if removeErr != nil {
+ return removeErr
+ }
+ if err := <-listErrCh; err != nil {
+ return fmt.Errorf("failed to list objects in bucket %s: %w", actualBucket, err)
+ }
+ if err := ctx.Err(); err != nil {
+ return err
}
// Only remove the actual bucket if not in single-bucket mode