Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feature: delete user all stuffs when delete user (#20010)

Wang Qi769d091d7ad5
diff --git a/internal/admin/service.go b/internal/admin/service.go
index a0570f987d..855089c238 100644
--- a/internal/admin/service.go
+++ b/internal/admin/service.go
@@ -74,6 +74,8 @@ type Service struct {
BillingSubscriptionDAO *dao.BillingSubscriptionDAO
MemoryDAO *dao.MemoryDAO
SearchDAO *dao.SearchDAO
+ deleteEngine engine.DocEngine
+ deleteStorage storage.Storage
}
// NewService create admin service
@@ -524,12 +526,11 @@ type DeleteUserResult struct {
// - *DeleteUserResult
// - error: error message
func (s *Service) DeleteUser(ctx context.Context, username string) (*DeleteUserResult, error) {
- result := &DeleteUserResult{
- Username: username,
- DeletedDetails: []string{fmt.Sprintf("Drop user: %s", username)},
- }
userList, err := s.userDAO.ListByEmail(ctx, dao.DB, username)
- if err != nil || len(userList) == 0 {
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user: %w", err)
+ }
+ if len(userList) == 0 {
return nil, fmt.Errorf("user '%s' not found", username)
}
@@ -549,191 +550,7 @@ func (s *Service) DeleteUser(ctx context.Context, username string) (*DeleteUserR
return nil, fmt.Errorf("user '%s' is admin account and cannot be deleted", username)
}
- // Get user-tenant relations
- tenants, err := s.userTenantDAO.GetByUserIDAll(ctx, dao.DB, user.ID)
- if err != nil {
- common.Warn("failed to get user-tenant relations", zap.Error(err))
- }
-
- // Find owned tenant (role = "owner")
- var ownedTenantID string
- for _, t := range tenants {
- if t.Role == "owner" {
- ownedTenantID = t.TenantID
- break
- }
- }
-
- // Start transaction for cascade delete
- tx := dao.DB.Begin()
- if tx.Error != nil {
- return nil, fmt.Errorf("failed to begin transaction: %w", tx.Error)
- }
-
- // Rollback helper function
- rollbackTx := func() {
- if rbErr := tx.Rollback(); rbErr.Error != nil {
- common.Error("failed to rollback transaction", rbErr.Error)
- }
- }
-
- result.DeletedDetails = append(result.DeletedDetails, "Start to delete owned tenant.")
- // Delete owned tenant data
- if ownedTenantID != "" {
- // 1. Get knowledge base IDs
- kbIDs, err := s.kbDAO.GetKBIDsByTenantIDSimple(ctx, tx, ownedTenantID)
- if err != nil {
- common.Warn("failed to get knowledge base IDs", zap.Error(err))
- }
-
- if len(kbIDs) > 0 {
- // 2. Get document IDs
- docIDs, err := s.documentDAO.GetAllDocIDsByKBIDs(ctx, tx, kbIDs)
- if err != nil {
- common.Warn("failed to get document IDs", zap.Error(err))
- }
-
- // 3. Delete tasks by document IDs
- if len(docIDs) > 0 {
- docIDList := make([]string, len(docIDs))
- for i, d := range docIDs {
- docIDList[i] = d["id"]
- }
- if delErr := tx.Unscoped().Where("doc_id IN ?", docIDList).Delete(&entity.Task{}); delErr.Error != nil {
- common.Warn("failed to delete tasks", zap.Error(delErr.Error))
- }
- }
-
- // 4. Delete documents
- if delErr := tx.Unscoped().Where("kb_id IN ?", kbIDs).Delete(&entity.Document{}); delErr.Error != nil {
- common.Warn("failed to delete documents", zap.Error(delErr.Error))
- }
-
- // 5. Delete knowledge bases
- if delErr := tx.Unscoped().Where("id IN ?", kbIDs).Delete(&entity.Knowledgebase{}); delErr.Error != nil {
- common.Warn("failed to delete knowledge bases", zap.Error(delErr.Error))
- }
- }
-
- // 6. Delete files
- if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.File{}); delErr.Error != nil {
- common.Warn("failed to delete files", zap.Error(delErr.Error))
- }
-
- // 7. Delete user canvas (agents)
- if delErr := tx.Unscoped().Where("user_id = ?", ownedTenantID).Delete(&entity.UserCanvas{}); delErr.Error != nil {
- common.Warn("failed to delete user canvas", zap.Error(delErr.Error))
- }
-
- // 8. Get dialog IDs
- var dialogIDs []string
- if pluckErr := tx.Model(&entity.Chat{}).Where("tenant_id = ?", ownedTenantID).Pluck("id", &dialogIDs); pluckErr.Error != nil {
- common.Warn("failed to get dialog IDs", zap.Error(pluckErr.Error))
- }
-
- // 9. Delete chat sessions
- if len(dialogIDs) > 0 {
- var sessionIDs []string
- if pluckErr := tx.Model(&entity.ChatSession{}).Where("dialog_id IN ?", dialogIDs).Pluck("id", &sessionIDs); pluckErr.Error != nil {
- common.Warn("failed to get chat session IDs", zap.Error(pluckErr.Error))
- }
- if len(sessionIDs) > 0 {
- if delErr := tx.Table("conversation_message").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
- common.Warn("failed to delete conversation messages", zap.Error(delErr.Error))
- }
- if delErr := tx.Table("conversation_reference").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
- common.Warn("failed to delete conversation references", zap.Error(delErr.Error))
- }
- }
- if delErr := tx.Unscoped().Where("dialog_id IN ?", dialogIDs).Delete(&entity.ChatSession{}); delErr.Error != nil {
- common.Warn("failed to delete chat sessions", zap.Error(delErr.Error))
- }
- }
-
- // 10. Delete chats/dialogs
- if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.Chat{}); delErr.Error != nil {
- common.Warn("failed to delete chats", zap.Error(delErr.Error))
- }
-
- // 11. Delete API tokens
- if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.APIToken{}); delErr.Error != nil {
- common.Warn("failed to delete API tokens", zap.Error(delErr.Error))
- }
-
- // 12. Delete API4Conversations
- if len(dialogIDs) > 0 {
- var sessionIDs []string
- if pluckErr := tx.Model(&entity.API4Conversation{}).Where("dialog_id IN ?", dialogIDs).Pluck("id", &sessionIDs); pluckErr.Error != nil {
- common.Warn("failed to get API conversation IDs", zap.Error(pluckErr.Error))
- }
- if len(sessionIDs) > 0 {
- if delErr := tx.Table("api_4_conversation_message").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
- common.Warn("failed to delete API conversation messages", zap.Error(delErr.Error))
- }
- if delErr := tx.Table("api_4_conversation_reference").Where("conversation_id IN ?", sessionIDs).Delete(map[string]interface{}{}); delErr.Error != nil {
- common.Warn("failed to delete API conversation references", zap.Error(delErr.Error))
- }
- }
- if delErr := tx.Unscoped().Where("dialog_id IN ?", dialogIDs).Delete(&entity.API4Conversation{}); delErr.Error != nil {
- common.Warn("failed to delete API4Conversations", zap.Error(delErr.Error))
- }
- }
-
- var tenantLLMCount int64
- tx.Model(&entity.TenantLLM{}).Where("tenant_id = ?", ownedTenantID).Count(&tenantLLMCount)
- result.TenantLLMCount = int(tenantLLMCount)
- result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d tenant-LLM records.", tenantLLMCount))
-
- result.LangfuseCount = 0
- result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d langfuse records.", result.LangfuseCount))
-
- metadataTableName := fmt.Sprintf("ragflow_doc_meta_%s", ownedTenantID[:32])
- result.MetadataTable = metadataTableName
- result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted metadata table %s.", metadataTableName))
-
- // 13. Delete tenant LLM configurations
- if delErr := tx.Unscoped().Where("tenant_id = ?", ownedTenantID).Delete(&entity.TenantLLM{}); delErr.Error != nil {
- common.Warn("failed to delete tenant LLM", zap.Error(delErr.Error))
- }
-
- var tenantCount int64
- tx.Model(&entity.Tenant{}).Where("id = ?", ownedTenantID).Count(&tenantCount)
- result.TenantCount = int(tenantCount)
- // 14. Delete tenant
- if delErr := tx.Unscoped().Where("id = ?", ownedTenantID).Delete(&entity.Tenant{}); delErr.Error != nil {
- common.Warn("failed to delete tenant", zap.Error(delErr.Error))
- }
- result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d tenant.", result.TenantCount))
- }
-
- var userTenantCount int64
- tx.Model(&entity.UserTenant{}).Where("user_id = ?", user.ID).Count(&userTenantCount)
- result.UserTenantCount = int(userTenantCount)
-
- // 15. Delete user-tenant relations
- if delErr := tx.Unscoped().Where("user_id = ?", user.ID).Delete(&entity.UserTenant{}); delErr.Error != nil {
- common.Warn("failed to delete user-tenant relations", zap.Error(delErr.Error))
- }
- result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d user-tenant records.", result.UserTenantCount))
-
- result.UserCount = 1
- // 16. Finally, hard delete user
- if delErr := tx.Unscoped().Where("id = ?", user.ID).Delete(&entity.User{}); delErr.Error != nil {
- rollbackTx()
- return nil, fmt.Errorf("failed to delete user: %w", delErr.Error)
- }
- result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d user.", result.UserCount))
-
- // Commit transaction
- if commitErr := tx.Commit(); commitErr.Error != nil {
- return nil, fmt.Errorf("failed to commit transaction: %w", commitErr.Error)
- }
-
- result.DeletedDetails = append(result.DeletedDetails, "Delete done!")
-
- common.Info("Delete user success with all related data", zap.String("username", username))
-
- return result, nil
+ return s.deleteUserData(ctx, user)
}
// ChangePassword change user password
diff --git a/internal/admin/user_delete.go b/internal/admin/user_delete.go
new file mode 100644
index 0000000000..4a69a8e713
--- /dev/null
+++ b/internal/admin/user_delete.go
@@ -0,0 +1,760 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package admin
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/knowledge_compile"
+ servicepkg "ragflow/internal/service"
+ "ragflow/internal/storage"
+
+ "go.uber.org/zap"
+ "gorm.io/gorm"
+)
+
+type userDeletionData struct {
+ ownedTenantID string
+ datasets []entity.Knowledgebase
+ documents []entity.Document
+ files []entity.File
+ memories []entity.Memory
+ spaces []entity.SkillSpace
+ datasetIDs []string
+ documentIDs []string
+ fileIDs []string
+ docTenants map[string]string
+ datasetNames map[string]string
+}
+
+func namedID(name, id string) string {
+ if name == "" {
+ return id
+ }
+ return fmt.Sprintf("%s (%s)", name, id)
+}
+
+func namedDocument(document entity.Document) string {
+ if document.Name == nil {
+ return document.ID
+ }
+ return namedID(*document.Name, document.ID)
+}
+
+func (s *Service) deleteUserData(ctx context.Context, user *entity.User) (*DeleteUserResult, error) {
+ data, err := loadUserDeletionData(ctx, user.ID)
+ if err != nil {
+ return nil, err
+ }
+ docEngine := s.deleteEngine
+ if docEngine == nil {
+ docEngine = engine.Get()
+ }
+ store := s.deleteStorage
+ if store == nil {
+ store = storage.GetStorageFactory().GetStorage()
+ }
+ if err := data.deleteExternalData(ctx, docEngine, store); err != nil {
+ return nil, err
+ }
+
+ result := &DeleteUserResult{
+ Username: user.Email,
+ DeletedDetails: []string{fmt.Sprintf("Drop user: %s", user.Email)},
+ }
+ if err := dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ return data.deleteDatabaseRows(ctx, tx, user.ID, result)
+ }); err != nil {
+ return nil, fmt.Errorf("failed to delete user data: %w", err)
+ }
+ datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
+ for _, id := range data.datasetIDs {
+ datasetIDs[id] = struct{}{}
+ }
+ publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
+ defer cancel()
+ publishWarning := false
+ for _, document := range data.documents {
+ if _, deletingDataset := datasetIDs[document.KbID]; deletingDataset {
+ continue
+ }
+ if err := publishCtx.Err(); err != nil {
+ common.Warn("stopped publishing document deletions", zap.Error(err))
+ publishWarning = true
+ break
+ }
+ if err := knowledge_compile.PublishDeleted(publishCtx, data.docTenants[document.KbID], document.KbID, document.ID, nil, nil); err != nil {
+ common.Warn("failed to publish document deletion", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.Error(err))
+ publishWarning = true
+ }
+ }
+ if publishWarning {
+ result.DeletedDetails = append(result.DeletedDetails, "- Warning: dataset artifact refresh could not be scheduled for some deleted documents.")
+ }
+ result.DeletedDetails = append(result.DeletedDetails, "Delete done!")
+ common.Info("Delete user success with all related data")
+ return result, nil
+}
+
+func loadUserDeletionData(ctx context.Context, userID string) (*userDeletionData, error) {
+ db := dao.DB.WithContext(ctx)
+ var tenants []entity.UserTenant
+ if err := db.Where("user_id = ? AND role = ?", userID, "owner").Find(&tenants).Error; err != nil {
+ return nil, fmt.Errorf("load owned tenant: %w", err)
+ }
+ if len(tenants) > 1 {
+ return nil, fmt.Errorf("user %s owns more than one tenant", userID)
+ }
+ data := &userDeletionData{}
+ if len(tenants) == 1 {
+ data.ownedTenantID = tenants[0].TenantID
+ }
+ datasetQuery := db.Model(&entity.Knowledgebase{}).Select("id", "tenant_id", "created_by", "name").Where("created_by = ?", userID)
+ if data.ownedTenantID != "" {
+ datasetQuery = datasetQuery.Or("tenant_id = ?", data.ownedTenantID)
+ }
+ if err := datasetQuery.Find(&data.datasets).Error; err != nil {
+ return nil, fmt.Errorf("load datasets: %w", err)
+ }
+ data.datasetNames = make(map[string]string, len(data.datasets))
+ for _, dataset := range data.datasets {
+ data.datasetIDs = append(data.datasetIDs, dataset.ID)
+ data.datasetNames[dataset.ID] = dataset.Name
+ }
+ if err := db.Model(&entity.Document{}).Select("id", "kb_id", "name", "location", "token_num", "chunk_num").Where("created_by = ?", userID).Find(&data.documents).Error; err != nil {
+ return nil, fmt.Errorf("load documents: %w", err)
+ }
+ seenDocuments := make(map[string]struct{}, len(data.documents))
+ for _, document := range data.documents {
+ seenDocuments[document.ID] = struct{}{}
+ }
+ for start := 0; start < len(data.datasetIDs); start += 1000 {
+ end := min(start+1000, len(data.datasetIDs))
+ var documents []entity.Document
+ if err := db.Model(&entity.Document{}).Select("id", "kb_id", "name", "location", "token_num", "chunk_num").Where("kb_id IN ?", data.datasetIDs[start:end]).Find(&documents).Error; err != nil {
+ return nil, fmt.Errorf("load dataset documents: %w", err)
+ }
+ for _, document := range documents {
+ if _, seen := seenDocuments[document.ID]; !seen {
+ data.documents = append(data.documents, document)
+ seenDocuments[document.ID] = struct{}{}
+ }
+ }
+ }
+ for _, document := range data.documents {
+ data.documentIDs = append(data.documentIDs, document.ID)
+ }
+ data.docTenants = make(map[string]string)
+ if len(data.documents) > 0 {
+ deletingDatasets := make(map[string]struct{}, len(data.datasetIDs))
+ for _, id := range data.datasetIDs {
+ deletingDatasets[id] = struct{}{}
+ }
+ remainingDatasetIDs := make(map[string]struct{})
+ for _, document := range data.documents {
+ if _, deleting := deletingDatasets[document.KbID]; !deleting {
+ remainingDatasetIDs[document.KbID] = struct{}{}
+ }
+ }
+ if len(remainingDatasetIDs) > 0 {
+ ids := make([]string, 0, len(remainingDatasetIDs))
+ for id := range remainingDatasetIDs {
+ ids = append(ids, id)
+ }
+ for start := 0; start < len(ids); start += 1000 {
+ end := min(start+1000, len(ids))
+ var datasets []entity.Knowledgebase
+ if err := db.Select("id", "tenant_id", "name").Where("id IN ?", ids[start:end]).Find(&datasets).Error; err != nil {
+ return nil, fmt.Errorf("load document tenants: %w", err)
+ }
+ for _, dataset := range datasets {
+ data.docTenants[dataset.ID] = dataset.TenantID
+ data.datasetNames[dataset.ID] = dataset.Name
+ }
+ }
+ if len(data.docTenants) != len(ids) {
+ return nil, fmt.Errorf("some documents refer to missing datasets")
+ }
+ }
+ }
+ fileQuery := db.Model(&entity.File{}).Select("id", "parent_id", "tenant_id", "name", "location", "type", "source_type").Where("created_by = ?", userID)
+ if data.ownedTenantID != "" {
+ fileQuery = fileQuery.Or("tenant_id = ?", data.ownedTenantID)
+ }
+ if err := fileQuery.Find(&data.files).Error; err != nil {
+ return nil, fmt.Errorf("load files: %w", err)
+ }
+ for _, file := range data.files {
+ data.fileIDs = append(data.fileIDs, file.ID)
+ }
+ if data.ownedTenantID != "" {
+ if err := db.Select("id", "tenant_id").Where("tenant_id = ?", data.ownedTenantID).Find(&data.memories).Error; err != nil {
+ return nil, fmt.Errorf("load memories: %w", err)
+ }
+ if err := db.Select("id", "tenant_id").Where("tenant_id = ?", data.ownedTenantID).Find(&data.spaces).Error; err != nil {
+ return nil, fmt.Errorf("load skill spaces: %w", err)
+ }
+ }
+ return data, nil
+}
+
+func (data *userDeletionData) deleteExternalData(ctx context.Context, docEngine engine.DocEngine, store storage.Storage) error {
+ // External stores cannot join the SQL transaction. Keep the user row until
+ // cleanup succeeds so a failed request can be retried.
+ if docEngine == nil && (data.ownedTenantID != "" || len(data.datasets) > 0 || len(data.documents) > 0) {
+ return fmt.Errorf("document engine is unavailable for user data cleanup")
+ }
+ if store == nil && (len(data.datasets) > 0 || len(data.documents) > 0 || len(data.files) > 0) {
+ return fmt.Errorf("storage is unavailable for user data cleanup")
+ }
+ datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
+ for _, id := range data.datasetIDs {
+ datasetIDs[id] = struct{}{}
+ }
+ for _, document := range data.documents {
+ if document.Location == nil || *document.Location == "" {
+ continue
+ }
+ exists, err := store.ObjectExists(ctx, document.KbID, *document.Location)
+ if err != nil {
+ return fmt.Errorf("check document %s: %w", document.ID, err)
+ }
+ if !exists {
+ common.Warn("Document object already missing", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
+ continue
+ }
+ if err := store.Remove(ctx, document.KbID, *document.Location); err != nil {
+ return fmt.Errorf("remove document %s: %w", document.ID, err)
+ }
+ common.Info("Removed document object", zap.String("document", namedDocument(document)), zap.String("document_id", document.ID), zap.String("dataset", namedID(data.datasetNames[document.KbID], document.KbID)), zap.String("bucket", document.KbID))
+ }
+ for _, dataset := range data.datasets {
+ exists, err := store.BucketExistsWithError(ctx, dataset.ID)
+ if err != nil {
+ return fmt.Errorf("check dataset bucket %s: %w", dataset.ID, err)
+ }
+ if !exists {
+ common.Warn("Dataset bucket already missing", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))
+ continue
+ }
+ if err := store.RemoveEmptyBucket(ctx, dataset.ID); err != nil {
+ common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID), zap.Error(err))
+ } else {
+ common.Info("Removed empty dataset bucket", zap.String("dataset", namedID(dataset.Name, dataset.ID)), zap.String("bucket", dataset.ID))
+ }
+ }
+ for _, file := range data.files {
+ if file.SourceType != string(entity.FileSourceKnowledgebase) && file.Location != nil && *file.Location != "" && file.Type != "folder" {
+ exists, err := store.ObjectExists(ctx, file.ParentID, *file.Location)
+ if err != nil {
+ return fmt.Errorf("check file %s: %w", file.ID, err)
+ }
+ if !exists {
+ common.Warn("File object already missing", zap.String("file", namedID(file.Name, file.ID)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
+ continue
+ }
+ if err := store.Remove(ctx, file.ParentID, *file.Location); err != nil {
+ return fmt.Errorf("remove file %s: %w", file.ID, err)
+ }
+ common.Info("Removed file object", zap.String("file", namedID(file.Name, file.ID)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
+ }
+ }
+ for _, file := range data.files {
+ if file.Type != "folder" || file.SourceType == string(entity.FileSourceKnowledgebase) || file.TenantID != data.ownedTenantID {
+ continue
+ }
+ exists, err := store.BucketExistsWithError(ctx, file.ID)
+ if err != nil {
+ return fmt.Errorf("check folder bucket %s: %w", file.ID, err)
+ }
+ if !exists {
+ common.Warn("Folder bucket already missing", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID))
+ continue
+ }
+ if err := store.RemoveEmptyBucket(ctx, file.ID); err != nil {
+ common.Warn("Failed to remove empty folder bucket", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID), zap.Error(err))
+ } else {
+ common.Info("Removed empty folder bucket", zap.String("folder", namedID(file.Name, file.ID)), zap.String("bucket", file.ID))
+ }
+ }
+ if docEngine == nil {
+ return nil
+ }
+ if data.ownedTenantID != "" {
+ ownedDatasetIDs := make([]string, 0)
+ for _, dataset := range data.datasets {
+ if dataset.TenantID == data.ownedTenantID {
+ ownedDatasetIDs = append(ownedDatasetIDs, dataset.ID)
+ }
+ }
+ if err := dropUserChunkStore(ctx, docEngine, servicepkg.IndexName(data.ownedTenantID), ownedDatasetIDs); err != nil {
+ return fmt.Errorf("remove tenant chunks: %w", err)
+ }
+ if err := dropUserChunkStore(ctx, docEngine, servicepkg.MemoryIndexName(data.ownedTenantID), memoryIDs(data.memories)); err != nil {
+ return fmt.Errorf("remove memory messages: %w", err)
+ }
+ if err := docEngine.DropMetadataStore(ctx, data.ownedTenantID); err != nil {
+ return fmt.Errorf("remove tenant metadata: %w", err)
+ }
+ }
+ for _, dataset := range data.datasets {
+ if dataset.TenantID != data.ownedTenantID {
+ indexName := servicepkg.IndexName(dataset.TenantID)
+ exists, err := docEngine.ChunkStoreExists(ctx, indexName, dataset.ID)
+ if err != nil {
+ return fmt.Errorf("check dataset chunks %s: %w", dataset.ID, err)
+ }
+ if !exists {
+ continue
+ }
+ if _, err := docEngine.DeleteChunks(ctx, map[string]interface{}{"kb_id": dataset.ID}, indexName, dataset.ID); err != nil {
+ return fmt.Errorf("remove dataset chunks %s: %w", dataset.ID, err)
+ }
+ }
+ }
+ documentsByDataset := make(map[string][]string)
+ for _, document := range data.documents {
+ if _, deletingDataset := datasetIDs[document.KbID]; !deletingDataset {
+ documentsByDataset[document.KbID] = append(documentsByDataset[document.KbID], document.ID)
+ }
+ }
+ for kbID, docIDs := range documentsByDataset {
+ tenantID := data.docTenants[kbID]
+ if tenantID == "" {
+ return fmt.Errorf("dataset %s for documents not found", kbID)
+ }
+ indexName := servicepkg.IndexName(tenantID)
+ exists, err := docEngine.ChunkStoreExists(ctx, indexName, kbID)
+ if err != nil {
+ return fmt.Errorf("check document chunks for dataset %s: %w", kbID, err)
+ }
+ if exists {
+ for start := 0; start < len(docIDs); start += 1000 {
+ end := min(start+1000, len(docIDs))
+ if _, err := docEngine.DeleteChunks(ctx, map[string]interface{}{"doc_id": docIDs[start:end]}, indexName, kbID); err != nil {
+ return fmt.Errorf("remove document chunks for dataset %s: %w", kbID, err)
+ }
+ }
+ }
+ metadataExists, err := docEngine.MetadataStoreExists(ctx, tenantID)
+ if err != nil {
+ return fmt.Errorf("check document metadata for dataset %s: %w", kbID, err)
+ }
+ if metadataExists {
+ for start := 0; start < len(docIDs); start += 1000 {
+ end := min(start+1000, len(docIDs))
+ if _, err := docEngine.DeleteMetadata(ctx, map[string]interface{}{"id": docIDs[start:end]}, tenantID); err != nil {
+ return fmt.Errorf("remove document metadata for dataset %s: %w", kbID, err)
+ }
+ }
+ }
+ }
+ for _, space := range data.spaces {
+ if err := docEngine.DropChunkStore(ctx, servicepkg.SkillIndexName(space.TenantID, space.ID), "skill"); err != nil {
+ return fmt.Errorf("remove skill index %s: %w", space.ID, err)
+ }
+ }
+ return nil
+}
+
+func dropUserChunkStore(ctx context.Context, docEngine engine.DocEngine, indexName string, ids []string) error {
+ if docEngine.GetType() == string(engine.EngineInfinity) {
+ for _, id := range ids {
+ if err := docEngine.DropChunkStore(ctx, indexName, id); err != nil {
+ return err
+ }
+ }
+ return nil
+ }
+ return docEngine.DropChunkStore(ctx, indexName, "")
+}
+
+func memoryIDs(memories []entity.Memory) []string {
+ ids := make([]string, 0, len(memories))
+ for _, memory := range memories {
+ ids = append(ids, memory.ID)
+ }
+ return ids
+}
+
+func (data *userDeletionData) deleteDatabaseRows(ctx context.Context, tx *gorm.DB, userID string, result *DeleteUserResult) error {
+ selectIDs := func(model interface{}, condition string, args ...interface{}) ([]string, error) {
+ var ids []string
+ err := tx.WithContext(ctx).Model(model).Where(condition, args...).Pluck("id", &ids).Error
+ return ids, err
+ }
+ remove := func(label string, model interface{}, condition string, args ...interface{}) (int64, error) {
+ deleted := tx.WithContext(ctx).Unscoped().Where(condition, args...).Delete(model)
+ if deleted.Error != nil {
+ return 0, fmt.Errorf("delete %s: %w", label, deleted.Error)
+ }
+ if deleted.RowsAffected > 0 {
+ result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d %s records.", deleted.RowsAffected, label))
+ }
+ return deleted.RowsAffected, nil
+ }
+ removeIDs := func(label string, model interface{}, column string, ids []string) error {
+ var total int64
+ for start := 0; start < len(ids); start += 1000 {
+ end := min(start+1000, len(ids))
+ deleted := tx.WithContext(ctx).Unscoped().Where(column+" IN ?", ids[start:end]).Delete(model)
+ if deleted.Error != nil {
+ return fmt.Errorf("delete %s: %w", label, deleted.Error)
+ }
+ total += deleted.RowsAffected
+ }
+ if total > 0 {
+ result.DeletedDetails = append(result.DeletedDetails, fmt.Sprintf("- Deleted %d %s records.", total, label))
+ }
+ return nil
+ }
+ owner := data.ownedTenantID
+ var chatIDs, canvasIDs, conversationIDs, apiConversationIDs []string
+ var evaluationDatasetIDs, evaluationRunIDs, connectorIDs, providerIDs, instanceIDs, modelIDs, groupIDs []string
+ var ingestionTaskIDs, memoryTaskIDs, commitIDs []string
+ var err error
+ if owner != "" {
+ if chatIDs, err = selectIDs(&entity.Chat{}, "tenant_id = ?", owner); err != nil {
+ return err
+ }
+ if evaluationDatasetIDs, err = selectIDs(&entity.EvaluationDataset{}, "tenant_id = ? OR created_by = ?", owner, userID); err != nil {
+ return err
+ }
+ if connectorIDs, err = selectIDs(&entity.Connector{}, "tenant_id = ?", owner); err != nil {
+ return err
+ }
+ if providerIDs, err = selectIDs(&entity.TenantModelProvider{}, "tenant_id = ?", owner); err != nil {
+ return err
+ }
+ } else if evaluationDatasetIDs, err = selectIDs(&entity.EvaluationDataset{}, "created_by = ?", userID); err != nil {
+ return err
+ }
+ if canvasIDs, err = selectIDs(&entity.UserCanvas{}, "user_id = ?", userID); err != nil {
+ return err
+ }
+ if len(chatIDs) > 0 {
+ conversationIDs, err = selectIDs(&entity.ChatSession{}, "dialog_id IN ? OR user_id = ?", chatIDs, userID)
+ if err != nil {
+ return err
+ }
+ } else {
+ conversationIDs, err = selectIDs(&entity.ChatSession{}, "user_id = ?", userID)
+ if err != nil {
+ return err
+ }
+ }
+ apiDialogIDs := append(append([]string{}, chatIDs...), canvasIDs...)
+ if len(apiDialogIDs) > 0 {
+ apiConversationIDs, err = selectIDs(&entity.API4Conversation{}, "dialog_id IN ? OR user_id = ?", apiDialogIDs, userID)
+ } else {
+ apiConversationIDs, err = selectIDs(&entity.API4Conversation{}, "user_id = ?", userID)
+ }
+ if err != nil {
+ return err
+ }
+ if len(evaluationDatasetIDs) > 0 {
+ evaluationRunIDs, err = selectIDs(&entity.EvaluationRun{}, "dataset_id IN ? OR created_by = ?", evaluationDatasetIDs, userID)
+ } else {
+ evaluationRunIDs, err = selectIDs(&entity.EvaluationRun{}, "created_by = ?", userID)
+ }
+ if err != nil {
+ return err
+ }
+ if len(providerIDs) > 0 {
+ if instanceIDs, err = selectIDs(&entity.TenantModelInstance{}, "provider_id IN ?", providerIDs); err != nil {
+ return err
+ }
+ if modelIDs, err = selectIDs(&entity.TenantModel{}, "provider_id IN ?", providerIDs); err != nil {
+ return err
+ }
+ if err = tx.Model(&entity.TenantModelGroupMapping{}).Where("provider_id IN ?", providerIDs).Distinct().Pluck("group_id", &groupIDs).Error; err != nil {
+ return err
+ }
+ }
+ if len(data.documentIDs) > 0 || len(data.datasetIDs) > 0 {
+ query := tx.Model(&entity.IngestionTask{}).Where("user_id = ?", userID)
+ if len(data.documentIDs) > 0 {
+ query = query.Or("document_id IN ?", data.documentIDs)
+ }
+ if len(data.datasetIDs) > 0 {
+ query = query.Or("dataset_id IN ?", data.datasetIDs)
+ }
+ err = query.Pluck("id", &ingestionTaskIDs).Error
+ } else {
+ ingestionTaskIDs, err = selectIDs(&entity.IngestionTask{}, "user_id = ?", userID)
+ }
+ if err != nil {
+ return err
+ }
+ if len(data.memories) > 0 {
+ if err = tx.Model(&entity.MemoryTask{}).Where("memory_id IN ?", memoryIDs(data.memories)).Pluck("task_id", &memoryTaskIDs).Error; err != nil {
+ return err
+ }
+ }
+ commitQuery := tx.Model(&entity.FileCommit{}).Where("author_id = ?", userID)
+ if len(data.fileIDs) > 0 {
+ commitQuery = commitQuery.Or("folder_id IN ?", data.fileIDs)
+ }
+ if err = commitQuery.Pluck("id", &commitIDs).Error; err != nil {
+ return err
+ }
+
+ for _, child := range []struct {
+ label string
+ model interface{}
+ ids []string
+ }{
+ {"conversation messages", &entity.ConversationMessage{}, conversationIDs},
+ {"conversation references", &entity.ConversationReference{}, conversationIDs},
+ {"API conversation messages", &entity.API4ConversationMessage{}, apiConversationIDs},
+ {"API conversation references", &entity.API4ConversationReference{}, apiConversationIDs},
+ } {
+ if err := removeIDs(child.label, child.model, "conversation_id", child.ids); err != nil {
+ return err
+ }
+ }
+ if err := removeIDs("conversations", &entity.ChatSession{}, "id", conversationIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("API conversations", &entity.API4Conversation{}, "id", apiConversationIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("chat channels", &entity.ChatChannel{}, "chat_id", chatIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("chats", &entity.Chat{}, "id", chatIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("agent versions", &entity.UserCanvasVersion{}, "user_canvas_id", canvasIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("agents", &entity.UserCanvas{}, "id", canvasIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("evaluation results", &entity.EvaluationResult{}, "run_id", evaluationRunIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("evaluation runs", &entity.EvaluationRun{}, "id", evaluationRunIDs); err != nil {
+ return err
+ }
+ if len(evaluationDatasetIDs) > 0 {
+ var caseIDs []string
+ if caseIDs, err = selectIDs(&entity.EvaluationCase{}, "dataset_id IN ?", evaluationDatasetIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("evaluation results", &entity.EvaluationResult{}, "case_id", caseIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("evaluation cases", &entity.EvaluationCase{}, "id", caseIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("evaluation datasets", &entity.EvaluationDataset{}, "id", evaluationDatasetIDs); err != nil {
+ return err
+ }
+ }
+ if err := removeIDs("ingestion task logs", &entity.IngestionTaskLog{}, "task_id", ingestionTaskIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("ingestion tasks", &entity.IngestionTask{}, "id", ingestionTaskIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("memory tasks", &entity.MemoryTask{}, "memory_id", memoryIDs(data.memories)); err != nil {
+ return err
+ }
+ if err := removeIDs("memory task records", &entity.Task{}, "id", memoryTaskIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("document tasks", &entity.Task{}, "doc_id", data.documentIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("document file links", &entity.File2Document{}, "document_id", data.documentIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("file document links", &entity.File2Document{}, "file_id", data.fileIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("documents", &entity.Document{}, "id", data.documentIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("file commit items", &entity.FileCommitItem{}, "commit_id", commitIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("file commit items", &entity.FileCommitItem{}, "file_id", data.fileIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("file commits", &entity.FileCommit{}, "id", commitIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("files", &entity.File{}, "id", data.fileIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("connector dataset links", &entity.Connector2Kb{}, "connector_id", connectorIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("connector dataset links", &entity.Connector2Kb{}, "kb_id", data.datasetIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("sync logs", &entity.SyncLogs{}, "connector_id", connectorIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("sync logs", &entity.SyncLogs{}, "kb_id", data.datasetIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("connectors", &entity.Connector{}, "id", connectorIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("knowledge compile records", &entity.KnowledgeCompileDataset{}, "dataset_id", data.datasetIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("wiki document records", &entity.WikiDocumentDirty{}, "document_id", data.documentIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("wiki document records", &entity.WikiDocumentDirty{}, "dataset_id", data.datasetIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("pipeline logs", &entity.PipelineOperationLog{}, "document_id", data.documentIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("pipeline logs", &entity.PipelineOperationLog{}, "kb_id", data.datasetIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("datasets", &entity.Knowledgebase{}, "id", data.datasetIDs); err != nil {
+ return err
+ }
+ datasetIDs := make(map[string]struct{}, len(data.datasetIDs))
+ for _, id := range data.datasetIDs {
+ datasetIDs[id] = struct{}{}
+ }
+ counts := make(map[string]struct{ documents, chunks, tokens int64 })
+ for _, document := range data.documents {
+ if _, deletingDataset := datasetIDs[document.KbID]; deletingDataset {
+ continue
+ }
+ count := counts[document.KbID]
+ count.documents++
+ count.chunks += document.ChunkNum
+ count.tokens += document.TokenNum
+ counts[document.KbID] = count
+ }
+ for kbID, count := range counts {
+ if err := tx.Model(&entity.Knowledgebase{}).Where("id = ?", kbID).Updates(map[string]interface{}{
+ "doc_num": gorm.Expr("CASE WHEN doc_num >= ? THEN doc_num - ? ELSE 0 END", count.documents, count.documents),
+ "chunk_num": gorm.Expr("CASE WHEN chunk_num >= ? THEN chunk_num - ? ELSE 0 END", count.chunks, count.chunks),
+ "token_num": gorm.Expr("CASE WHEN token_num >= ? THEN token_num - ? ELSE 0 END", count.tokens, count.tokens),
+ }).Error; err != nil {
+ return fmt.Errorf("update dataset counters: %w", err)
+ }
+ }
+ if len(groupIDs) > 0 {
+ if err := removeIDs("model group mappings", &entity.TenantModelGroupMapping{}, "provider_id", providerIDs); err != nil {
+ return err
+ }
+ for _, groupID := range groupIDs {
+ var remaining int64
+ if err := tx.Model(&entity.TenantModelGroupMapping{}).Where("group_id = ?", groupID).Count(&remaining).Error; err != nil {
+ return err
+ }
+ if remaining == 0 {
+ if _, err := remove("model groups", &entity.TenantModelGroup{}, "id = ?", groupID); err != nil {
+ return err
+ }
+ }
+ }
+ }
+ if err := removeIDs("models", &entity.TenantModel{}, "id", modelIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("model instances", &entity.TenantModelInstance{}, "id", instanceIDs); err != nil {
+ return err
+ }
+ if err := removeIDs("model providers", &entity.TenantModelProvider{}, "id", providerIDs); err != nil {
+ return err
+ }
+ if owner != "" {
+ for _, item := range []struct {
+ label string
+ model interface{}
+ }{
+ {"chat channels", &entity.ChatChannel{}},
+ {"searches", &entity.Search{}},
+ {"memories", &entity.Memory{}},
+ {"MCP servers", &entity.MCPServer{}},
+ {"skill search configurations", &entity.SkillSearchConfig{}},
+ {"skill spaces", &entity.SkillSpace{}},
+ {"compilation templates", &entity.CompilationTemplate{}},
+ {"compilation template groups", &entity.CompilationTemplateGroup{}},
+ {"pipeline logs", &entity.PipelineOperationLog{}},
+ {"knowledge compile records", &entity.KnowledgeCompileDataset{}},
+ {"wiki document records", &entity.WikiDocumentDirty{}},
+ {"API tokens", &entity.APIToken{}},
+ } {
+ if _, err := remove(item.label, item.model, "tenant_id = ?", owner); err != nil {
+ return err
+ }
+ }
+ tenantLLMCount, err := remove("tenant LLMs", &entity.TenantLLM{}, "tenant_id = ?", owner)
+ if err != nil {
+ return err
+ }
+ result.TenantLLMCount = int(tenantLLMCount)
+ langfuseCount, err := remove("Langfuse credentials", &entity.TenantLangfuse{}, "tenant_id = ?", owner)
+ if err != nil {
+ return err
+ }
+ result.LangfuseCount = int(langfuseCount)
+ result.MetadataTable = servicepkg.BuildMetadataIndexName(owner)
+ ownerMemberships, err := remove("tenant memberships", &entity.UserTenant{}, "tenant_id = ?", owner)
+ if err != nil {
+ return err
+ }
+ result.UserTenantCount += int(ownerMemberships)
+ count, err := remove("tenants", &entity.Tenant{}, "id = ?", owner)
+ if err != nil {
+ return err
+ }
+ result.TenantCount = int(count)
+ }
+ if _, err := remove("searches", &entity.Search{}, "created_by = ?", userID); err != nil {
+ return err
+ }
+ invitationCondition := "user_id = ?"
+ invitationArgs := []interface{}{userID}
+ if owner != "" {
+ invitationCondition += " OR tenant_id = ?"
+ invitationArgs = append(invitationArgs, owner)
+ }
+ if _, err := remove("invitations", &entity.InvitationCode{}, invitationCondition, invitationArgs...); err != nil {
+ return err
+ }
+ count, err := remove("user tenant memberships", &entity.UserTenant{}, "user_id = ?", userID)
+ if err != nil {
+ return err
+ }
+ result.UserTenantCount += int(count)
+ count, err = remove("users", &entity.User{}, "id = ?", userID)
+ if err != nil {
+ return err
+ }
+ result.UserCount = int(count)
+ return nil
+}
diff --git a/internal/admin/user_delete_test.go b/internal/admin/user_delete_test.go
new file mode 100644
index 0000000000..ca1feba37b
--- /dev/null
+++ b/internal/admin/user_delete_test.go
@@ -0,0 +1,384 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package admin
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "strings"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/engine"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
+ "ragflow/internal/storage"
+
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "go.uber.org/zap/zaptest/observer"
+ "gorm.io/gorm"
+)
+
+type deletionEngine struct {
+ engine.DocEngine
+ dropped []string
+ deleted []string
+}
+
+func (e *deletionEngine) GetType() string { return string(engine.EngineElasticsearch) }
+func (e *deletionEngine) DropChunkStore(_ context.Context, name, _ string) error {
+ e.dropped = append(e.dropped, name)
+ return nil
+}
+func (e *deletionEngine) DropMetadataStore(_ context.Context, tenantID string) error {
+ e.dropped = append(e.dropped, "metadata:"+tenantID)
+ return nil
+}
+func (e *deletionEngine) ChunkStoreExists(context.Context, string, string) (bool, error) {
+ return true, nil
+}
+func (e *deletionEngine) MetadataStoreExists(context.Context, string) (bool, error) {
+ return true, nil
+}
+func (e *deletionEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, _, _ string) (int64, error) {
+ for key, value := range condition {
+ e.deleted = append(e.deleted, fmt.Sprintf("%s:%v", key, value))
+ }
+ return 1, nil
+}
+func (e *deletionEngine) DeleteMetadata(_ context.Context, condition map[string]interface{}, _ string) (int64, error) {
+ for key, value := range condition {
+ e.deleted = append(e.deleted, fmt.Sprintf("metadata:%s:%v", key, value))
+ }
+ return 1, nil
+}
+
+type deletionStorage struct {
+ storage.Storage
+ buckets []string
+ files []string
+ err error
+ bucketErr error
+ missing bool
+ checkErr error
+}
+
+func (s *deletionStorage) RemoveBucket(_ context.Context, bucket string) error {
+ if s.err != nil {
+ return s.err
+ }
+ s.buckets = append(s.buckets, bucket)
+ return nil
+}
+func (s *deletionStorage) RemoveEmptyBucket(_ context.Context, bucket string) error {
+ if s.bucketErr != nil {
+ return s.bucketErr
+ }
+ s.buckets = append(s.buckets, bucket)
+ return nil
+}
+func (s *deletionStorage) Remove(_ context.Context, bucket, name string, _ ...string) error {
+ if s.err != nil {
+ return s.err
+ }
+ s.files = append(s.files, bucket+"/"+name)
+ return nil
+}
+func (s *deletionStorage) ObjectExists(_ context.Context, bucket, name string) (bool, error) {
+ if s.checkErr != nil {
+ return false, s.checkErr
+ }
+ return !s.missing, nil
+}
+func (s *deletionStorage) BucketExistsWithError(_ context.Context, bucket string) (bool, error) {
+ if s.checkErr != nil {
+ return false, s.checkErr
+ }
+ return !s.missing, nil
+}
+
+func setupUserDeletionDB(t *testing.T) *gorm.DB {
+ t.Helper()
+ db := testutil.SetupTestDB(t,
+ &entity.User{}, &entity.Tenant{}, &entity.UserTenant{},
+ &entity.Knowledgebase{}, &entity.Document{}, &entity.Task{}, &entity.File{}, &entity.File2Document{},
+ &entity.Chat{}, &entity.ChatSession{}, &entity.ConversationMessage{}, &entity.ConversationReference{},
+ &entity.API4Conversation{}, &entity.API4ConversationMessage{}, &entity.API4ConversationReference{},
+ &entity.ChatChannel{}, &entity.UserCanvas{}, &entity.UserCanvasVersion{},
+ &entity.Search{}, &entity.Memory{}, &entity.MemoryTask{}, &entity.MCPServer{},
+ &entity.SkillSpace{}, &entity.SkillSearchConfig{},
+ &entity.CompilationTemplate{}, &entity.CompilationTemplateGroup{},
+ &entity.EvaluationDataset{}, &entity.EvaluationCase{}, &entity.EvaluationRun{}, &entity.EvaluationResult{},
+ &entity.Connector{}, &entity.Connector2Kb{}, &entity.SyncLogs{},
+ &entity.KnowledgeCompileDataset{}, &entity.WikiDocumentDirty{}, &entity.PipelineOperationLog{},
+ &entity.IngestionTask{}, &entity.IngestionTaskLog{}, &entity.FileCommit{}, &entity.FileCommitItem{},
+ &entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{},
+ &entity.TenantModelGroup{}, &entity.TenantModelGroupMapping{},
+ &entity.TenantLLM{}, &entity.TenantLangfuse{}, &entity.APIToken{}, &entity.InvitationCode{},
+ )
+ t.Cleanup(testutil.ReplaceDBForTest(t, db))
+ return db
+}
+
+func TestDeleteUserRemovesOwnedDataAndJoinedDocuments(t *testing.T) {
+ db := setupUserDeletionDB(t)
+ insert := func(query string, args ...interface{}) {
+ t.Helper()
+ if err := db.Exec(query, args...).Error; err != nil {
+ t.Fatalf("insert fixture: %v", err)
+ }
+ }
+ insert("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0")
+ insert("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-2", "[email protected]", "Other", "1", "1", "0")
+ insert("INSERT INTO tenant (id, llm_id, embd_id, asr_id, img2txt_id, rerank_id, parser_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "user-1", "llm", "embd", "asr", "img", "rerank", "general")
+ insert("INSERT INTO user_tenant (id, user_id, tenant_id, role, invited_by) VALUES (?, ?, ?, ?, ?)", "owner", "user-1", "user-1", "owner", "user-1")
+ insert("INSERT INTO user_tenant (id, user_id, tenant_id, role, invited_by) VALUES (?, ?, ?, ?, ?)", "joined", "user-1", "team", "normal", "user-2")
+ insert("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id, doc_num, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "own-kb", "user-1", "Own", "embd", "me", "user-1", "general", 1, 2, 3)
+ insert("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id, doc_num, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "team-kb", "team", "Team", "embd", "team", "user-2", "general", 2, 5, 7)
+ insert("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "own-doc", "own-kb", "Owned document", "general", "{}", "pdf", "user-1", "pdf", "own.pdf", 2, 3)
+ insert("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "joined-doc", "team-kb", "Joined document", "general", "{}", "pdf", "user-1", "pdf", "joined.pdf", 2, 3)
+ insert("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, token_num, chunk_num) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", "other-doc", "team-kb", "general", "{}", "pdf", "user-2", "pdf", 3, 4)
+ insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "own-file", "own-folder", "user-1", "user-1", "own.pdf", "own.pdf", "file")
+ insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, type) VALUES (?, ?, ?, ?, ?, ?)", "own-folder", "own-folder", "user-1", "user-1", "/", "folder")
+ insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type, source_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "kb-file", "kb-folder", "user-1", "user-1", "own.pdf", "own.pdf", "pdf", "knowledgebase")
+ insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "joined-file", "team-folder", "team", "user-1", "joined.pdf", "joined.pdf", "file")
+ insert("INSERT INTO file (id, parent_id, tenant_id, created_by, name, location, type) VALUES (?, ?, ?, ?, ?, ?, ?)", "other-file", "team-folder", "team", "user-2", "other.pdf", "other.pdf", "file")
+ insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "own-link", "own-file", "own-doc")
+ insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "joined-link", "joined-file", "joined-doc")
+ insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "shared-link", "other-file", "joined-doc")
+ insert("INSERT INTO file2document (id, file_id, document_id) VALUES (?, ?, ?)", "other-link", "other-file", "other-doc")
+ insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "own-search", "user-1", "Own", "user-1", "{}")
+ insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "joined-search", "team", "Joined", "user-1", "{}")
+ insert("INSERT INTO search (id, tenant_id, name, created_by, search_config) VALUES (?, ?, ?, ?, ?)", "other-search", "team", "Other", "user-2", "{}")
+ insert("INSERT INTO dialog (id, tenant_id, llm_id, llm_setting, prompt_config, do_refer, kb_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "own-chat", "user-1", "llm", "{}", "{}", "1", "[]")
+ insert("INSERT INTO dialog (id, tenant_id, llm_id, llm_setting, prompt_config, do_refer, kb_ids) VALUES (?, ?, ?, ?, ?, ?, ?)", "team-chat", "team", "llm", "{}", "{}", "1", "[]")
+ insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "own-conversation", "own-chat", "user-1")
+ insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "joined-conversation", "team-chat", "user-1")
+ insert("INSERT INTO conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "other-conversation", "team-chat", "user-2")
+ insert("INSERT INTO conversation_message (conversation_id, position, metadata) VALUES (?, ?, ?)", "own-conversation", 0, "{}")
+ insert("INSERT INTO conversation_reference (conversation_id, position, reference) VALUES (?, ?, ?)", "own-conversation", 0, "{}")
+ insert("INSERT INTO api_4_conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "own-api-conversation", "own-chat", "user-1")
+ insert("INSERT INTO api_4_conversation_message (conversation_id, position, metadata) VALUES (?, ?, ?)", "own-api-conversation", 0, "{}")
+ insert("INSERT INTO api_4_conversation_reference (conversation_id, position, reference) VALUES (?, ?, ?)", "own-api-conversation", 0, "{}")
+ insert("INSERT INTO user_canvas (id, user_id, tags, canvas_category) VALUES (?, ?, ?, ?)", "agent", "user-1", "", "agent_canvas")
+ insert("INSERT INTO user_canvas_version (id, user_canvas_id) VALUES (?, ?)", "agent-version", "agent")
+ insert("INSERT INTO api_4_conversation (id, dialog_id, user_id) VALUES (?, ?, ?)", "agent-conversation", "agent", "external-user")
+ insert("INSERT INTO chat_channel (id, tenant_id, name, channel, config) VALUES (?, ?, ?, ?, ?)", "channel", "user-1", "Channel", "telegram", "{}")
+ insert("INSERT INTO memory (id, tenant_id, name, embd_id, llm_id, permissions, forgetting_policy) VALUES (?, ?, ?, ?, ?, ?, ?)", "memory", "user-1", "Memory", "embd", "llm", "me", "FIFO")
+ insert("INSERT INTO tenant_model_provider (id, provider_name, tenant_id) VALUES (?, ?, ?)", "provider", "custom", "user-1")
+ insert("INSERT INTO tenant_model_instance (id, instance_name, provider_id, api_key) VALUES (?, ?, ?, ?)", "instance", "custom", "provider", "secret")
+ insert("INSERT INTO tenant_model (id, model_name, provider_id, instance_id, model_type) VALUES (?, ?, ?, ?, ?)", "model", "model", "provider", "instance", 1)
+ insert("INSERT INTO tenant_model_group (id, group_type, strategy) VALUES (?, ?, ?)", "group", "chat", "weighted")
+ insert("INSERT INTO tenant_model_group_mapping (group_id, provider_id, instance_id, model_id) VALUES (?, ?, ?, ?)", "group", "provider", "instance", "model")
+
+ docEngine := &deletionEngine{}
+ store := &deletionStorage{}
+ core, logs := observer.New(zapcore.InfoLevel)
+ oldLogger := common.Logger
+ common.Logger = zap.New(core)
+ t.Cleanup(func() { common.Logger = oldLogger })
+ service := NewService()
+ service.deleteEngine = docEngine
+ service.deleteStorage = store
+ result, err := service.DeleteUser(t.Context(), "[email protected]")
+ if err != nil {
+ t.Fatalf("DeleteUser: %v", err)
+ }
+ if result.UserCount != 1 || result.TenantCount != 1 {
+ t.Fatalf("unexpected deletion result: %+v", result)
+ }
+ for _, item := range [][2]string{
+ {"knowledgebase", "own-kb"}, {"document", "joined-doc"}, {"file", "own-file"}, {"file", "joined-file"},
+ {"search", "own-search"}, {"search", "joined-search"}, {"dialog", "own-chat"},
+ {"conversation", "own-conversation"}, {"conversation", "joined-conversation"}, {"api_4_conversation", "own-api-conversation"}, {"api_4_conversation", "agent-conversation"},
+ {"user_canvas", "agent"}, {"user_canvas_version", "agent-version"}, {"chat_channel", "channel"},
+ {"memory", "memory"}, {"tenant_model_provider", "provider"}, {"tenant_model_instance", "instance"},
+ {"tenant_model", "model"}, {"tenant_model_group", "group"},
+ } {
+ var count int64
+ if err := db.Table(item[0]).Where("id = ?", item[1]).Count(&count).Error; err != nil {
+ t.Fatal(err)
+ }
+ if count != 0 {
+ t.Fatalf("%s retained deleted resource %s", item[0], item[1])
+ }
+ }
+ for _, table := range []string{"conversation_message", "conversation_reference", "api_4_conversation_message", "api_4_conversation_reference", "file2document", "tenant_model_group_mapping"} {
+ var count int64
+ want := int64(0)
+ if table == "file2document" {
+ want = 1
+ }
+ if err := db.Table(table).Count(&count).Error; err != nil || count != want {
+ t.Fatalf("dependent rows remain in %s: count=%d, err=%v", table, count, err)
+ }
+ }
+ for _, tableID := range [][2]string{{"document", "other-doc"}, {"file", "other-file"}, {"search", "other-search"}, {"knowledgebase", "team-kb"}, {"dialog", "team-chat"}, {"conversation", "other-conversation"}} {
+ var count int64
+ if err := db.Table(tableID[0]).Where("id = ?", tableID[1]).Count(&count).Error; err != nil || count != 1 {
+ t.Fatalf("shared resource %s/%s missing: count=%d, err=%v", tableID[0], tableID[1], count, err)
+ }
+ }
+ var team entity.Knowledgebase
+ if err := db.First(&team, "id = ?", "team-kb").Error; err != nil || team.DocNum != 1 || team.TokenNum != 3 || team.ChunkNum != 4 {
+ t.Fatalf("joined dataset counters = %+v, err=%v", team, err)
+ }
+ if !slices.Equal(store.buckets, []string{"own-kb", "own-folder"}) || !slices.Equal(store.files, []string{"own-kb/own.pdf", "team-kb/joined.pdf", "own-folder/own.pdf", "team-folder/joined.pdf"}) {
+ t.Fatalf("storage cleanup = buckets %v, files %v", store.buckets, store.files)
+ }
+ if !slices.Contains(docEngine.dropped, "ragflow_user-1") || !slices.Contains(docEngine.dropped, "memory_user-1") || !slices.Contains(docEngine.deleted, "doc_id:[joined-doc]") || !slices.Contains(docEngine.deleted, "metadata:id:[joined-doc]") {
+ t.Fatalf("index cleanup = dropped %v, deleted %v", docEngine.dropped, docEngine.deleted)
+ }
+ for _, check := range []struct {
+ message string
+ field string
+ want string
+ }{
+ {"Removed document object", "document", "Joined document (joined-doc)"},
+ {"Removed document object", "dataset", "Team (team-kb)"},
+ {"Removed file object", "file", "own.pdf (own-file)"},
+ {"Removed empty folder bucket", "folder", "/ (own-folder)"},
+ } {
+ matched := false
+ for _, entry := range logs.FilterMessage(check.message).All() {
+ if entry.ContextMap()[check.field] == check.want {
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ t.Errorf("missing %s log with %s=%q", check.message, check.field, check.want)
+ }
+ }
+}
+
+func TestDeleteUserKeepsUserWhenExternalCleanupFails(t *testing.T) {
+ db := setupUserDeletionDB(t)
+ if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
+ t.Fatal(err)
+ }
+ service := NewService()
+ service.deleteEngine = &deletionEngine{}
+ service.deleteStorage = &deletionStorage{err: errors.New("storage unavailable")}
+ if _, err := service.DeleteUser(t.Context(), "[email protected]"); err == nil || !strings.Contains(err.Error(), "storage unavailable") {
+ t.Fatalf("DeleteUser error = %v", err)
+ }
+ var count int64
+ if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 1 {
+ t.Fatalf("user was deleted after external failure: count=%d, err=%v", count, err)
+ }
+}
+
+func TestDeleteUserContinuesWhenEmptyBucketRemovalFails(t *testing.T) {
+ db := setupUserDeletionDB(t)
+ if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
+ t.Fatal(err)
+ }
+ service := NewService()
+ service.deleteEngine = &deletionEngine{}
+ service.deleteStorage = &deletionStorage{bucketErr: errors.New("bucket is not empty")}
+ if _, err := service.DeleteUser(t.Context(), "[email protected]"); err != nil {
+ t.Fatalf("DeleteUser after bucket removal failure: %v", err)
+ }
+ var count int64
+ if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 0 {
+ t.Fatalf("user retained after bucket failure: count=%d, err=%v", count, err)
+ }
+}
+
+func TestDeleteUserContinuesWhenStorageItemsAreMissing(t *testing.T) {
+ db := setupUserDeletionDB(t)
+ if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
+ t.Fatal(err)
+ }
+ store := &deletionStorage{missing: true}
+ service := NewService()
+ service.deleteEngine = &deletionEngine{}
+ service.deleteStorage = store
+ if _, err := service.DeleteUser(t.Context(), "[email protected]"); err != nil {
+ t.Fatalf("DeleteUser with missing storage items: %v", err)
+ }
+ if len(store.files) != 0 || len(store.buckets) != 0 {
+ t.Fatalf("storage deletion attempted for missing items: files=%v buckets=%v", store.files, store.buckets)
+ }
+}
+
+func TestDeleteUserStopsOnStorageCheckError(t *testing.T) {
+ db := setupUserDeletionDB(t)
+ if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO knowledgebase (id, tenant_id, name, embd_id, permission, created_by, parser_id) VALUES (?, ?, ?, ?, ?, ?, ?)", "dataset", "team", "Dataset", "embd", "me", "user-1", "general").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc", "dataset", "general", "{}", "pdf", "user-1", "pdf", "doc.pdf").Error; err != nil {
+ t.Fatal(err)
+ }
+ service := NewService()
+ service.deleteEngine = &deletionEngine{}
+ service.deleteStorage = &deletionStorage{checkErr: errors.New("storage unavailable")}
+ if _, err := service.DeleteUser(t.Context(), "[email protected]"); err == nil || !strings.Contains(err.Error(), "storage unavailable") {
+ t.Fatalf("DeleteUser error = %v", err)
+ }
+ var count int64
+ if err := db.Model(&entity.User{}).Where("id = ?", "user-1").Count(&count).Error; err != nil || count != 1 {
+ t.Fatalf("user deleted after storage check failed: count=%d, err=%v", count, err)
+ }
+}
+
+func TestDeleteUserRollsBackDatabaseCleanup(t *testing.T) {
+ db := setupUserDeletionDB(t)
+ if err := db.Exec("INSERT INTO user (id, email, nickname, is_active, is_authenticated, is_anonymous) VALUES (?, ?, ?, ?, ?, ?)", "user-1", "[email protected]", "User", "0", "1", "0").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO user_canvas (id, user_id, tags, canvas_category) VALUES (?, ?, ?, ?)", "agent", "user-1", "", "agent_canvas").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Exec("INSERT INTO user_canvas_version (id, user_canvas_id) VALUES (?, ?)", "version", "agent").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Migrator().DropTable(&entity.Search{}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := NewService().DeleteUser(t.Context(), "[email protected]"); err == nil {
+ t.Fatal("expected SQL cleanup failure")
+ }
+ for _, item := range [][2]string{{"user", "user-1"}, {"user_canvas", "agent"}, {"user_canvas_version", "version"}} {
+ var count int64
+ if err := db.Table(item[0]).Where("id = ?", item[1]).Count(&count).Error; err != nil || count != 1 {
+ t.Fatalf("transaction did not preserve %s/%s: count=%d, err=%v", item[0], item[1], count, err)
+ }
+ }
+}
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 558ecc76fa..e9cfbd07c0 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -1339,8 +1339,15 @@ func (s *chunkImageStorage) ListObjects(ctx context.Context, bucket string, tena
func (s *chunkImageStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
return "", nil
}
-func (s *chunkImageStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
-func (s *chunkImageStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
+func (s *chunkImageStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
+func (s *chunkImageStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
+func (s *chunkImageStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error { return nil }
+func (s *chunkImageStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
+ return s.exists, nil
+}
+func (s *chunkImageStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ return true, nil
+}
func (s *chunkImageStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
return false
}
diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.go
index 72c9a46690..c53a88872f 100644
--- a/internal/service/dataset/crud.go
+++ b/internal/service/dataset/crud.go
@@ -318,19 +318,17 @@ func (d *DatasetService) DeleteDatasets(ctx context.Context, ids []string, delet
}
successCount := 0
- errorsList := make([]string, 0)
for _, kb := range kbs {
if err := d.deleteDataset(ctx, tenantID, kb); err != nil {
- errorsList = append(errorsList, err.Error())
- common.Warn("deleteDataset failed", zap.String("kb_id", kb.ID), zap.Error(err))
- continue
+ common.Warn("deleteDataset failed", zap.String("dataset", datasetNameAndID(kb)), zap.String("kb_id", kb.ID), zap.Error(err))
+ return nil, common.CodeServerError, err
}
successCount++
}
return map[string]interface{}{
"success_count": successCount,
- "errors": errorsList,
+ "errors": []string{},
}, common.CodeSuccess, nil
}
@@ -339,6 +337,7 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
if storageImpl == nil {
return fmt.Errorf("storage not initialized")
}
+ dataset := datasetNameAndID(kb)
// Collect document IDs first so engine cleanup can run before the
// transaction (engine ops are not transactional).
@@ -346,6 +345,38 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
if err := dao.DB.Where("kb_id = ?", kb.ID).Find(&documents).Error; err != nil {
return fmt.Errorf("delete dataset error for %s", kb.ID)
}
+ cleanupCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+ defer cancel()
+ for _, document := range documents {
+ if document.Location == nil || *document.Location == "" {
+ continue
+ }
+ exists, err := storageImpl.ObjectExists(cleanupCtx, kb.ID, *document.Location)
+ if err != nil {
+ return fmt.Errorf("check document %s in dataset %s: %w", document.ID, kb.ID, err)
+ }
+ if !exists {
+ common.Warn("Dataset document object already missing", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))
+ continue
+ }
+ if err := storageImpl.Remove(cleanupCtx, kb.ID, *document.Location); err != nil {
+ return fmt.Errorf("remove document %s from dataset %s: %w", document.ID, kb.ID, err)
+ }
+ common.Info("Removed dataset document object", zap.String("document", documentNameAndID(document)), zap.String("document_id", document.ID), zap.String("dataset", dataset), zap.String("bucket", kb.ID))
+ }
+ exists, err := storageImpl.BucketExistsWithError(cleanupCtx, kb.ID)
+ if err != nil {
+ return fmt.Errorf("check dataset bucket %s: %w", kb.ID, err)
+ }
+ if !exists {
+ common.Warn("Dataset bucket already missing", zap.String("dataset", dataset), zap.String("bucket", kb.ID))
+ } else {
+ if err := storageImpl.RemoveEmptyBucket(cleanupCtx, kb.ID); err != nil {
+ common.Warn("Unable to remove empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID), zap.Error(err))
+ } else {
+ common.Info("Removed empty dataset bucket", zap.String("dataset", dataset), zap.String("bucket", kb.ID))
+ }
+ }
docIDs := extractDocIDs(documents)
if len(docIDs) > 0 {
d.deleteDatasetEngineData(ctx, kb, docIDs)
@@ -397,15 +428,24 @@ func (d *DatasetService) deleteDataset(ctx context.Context, tenantID string, kb
}); err != nil {
return err
}
- // Dataset uploads and generated objects are stored under the KB ID.
- cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Minute)
- defer cancel()
- if err := storageImpl.RemoveBucket(cleanupCtx, kb.ID); err != nil {
- return fmt.Errorf("dataset %s deleted but storage cleanup failed: %w", kb.ID, err)
- }
+ common.Info("Deleted dataset", zap.String("dataset", dataset), zap.String("kb_id", kb.ID))
return nil
}
+func documentNameAndID(document entity.Document) string {
+ if document.Name == nil || *document.Name == "" {
+ return document.ID
+ }
+ return fmt.Sprintf("%s (%s)", *document.Name, document.ID)
+}
+
+func datasetNameAndID(kb *entity.Knowledgebase) string {
+ if kb.Name == "" {
+ return kb.ID
+ }
+ return fmt.Sprintf("%s (%s)", kb.Name, kb.ID)
+}
+
func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, page, pageSize int, terms []dao.OrderTerm, keywords string, ownerIDs []string, parserID, userID string, ids []string) ([]map[string]interface{}, int64, common.ErrorCode, error) {
id = strings.TrimSpace(id)
if id != "" && len(ids) > 0 {
diff --git a/internal/service/dataset/crud_delete_test.go b/internal/service/dataset/crud_delete_test.go
new file mode 100644
index 0000000000..cac67493b8
--- /dev/null
+++ b/internal/service/dataset/crud_delete_test.go
@@ -0,0 +1,103 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dataset
+
+import (
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/storage"
+
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "go.uber.org/zap/zaptest/observer"
+)
+
+func TestDeleteDatasetLeavesNonemptyBucket(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Dataset")
+ if err := db.Exec("INSERT INTO document (id, kb_id, name, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", "doc-1", "kb-1", "Document", "general", "{}", "pdf", "tenant-1", "pdf", "tracked").Error; err != nil {
+ t.Fatal(err)
+ }
+ store := storage.NewMemoryStorage()
+ if err := store.Put(t.Context(), "kb-1", "tracked", []byte("content")); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Put(t.Context(), "kb-1", "untracked", []byte("content")); err != nil {
+ t.Fatal(err)
+ }
+ factory := storage.GetStorageFactory()
+ previous := factory.GetStorage()
+ factory.SetStorage(store)
+ t.Cleanup(func() { factory.SetStorage(previous) })
+ core, logs := observer.New(zapcore.InfoLevel)
+ previousLogger := common.Logger
+ common.Logger = zap.New(core)
+ t.Cleanup(func() { common.Logger = previousLogger })
+
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ service := testDatasetUpdateService(t)
+ if err := service.deleteDataset(t.Context(), "tenant-1", kb); err != nil {
+ t.Fatalf("delete dataset with nonempty bucket: %v", err)
+ }
+ if !store.ObjExist(t.Context(), "kb-1", "untracked") {
+ t.Fatal("dataset deletion removed an untracked object")
+ }
+ if store.ObjExist(t.Context(), "kb-1", "tracked") {
+ t.Fatal("dataset deletion retained a document object")
+ }
+ var count int64
+ if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Count(&count).Error; err != nil || count != 0 {
+ t.Fatalf("dataset retained after storage failure: count=%d, err=%v", count, err)
+ }
+ if entries := logs.FilterMessage("Removed dataset document object").All(); len(entries) != 1 || entries[0].ContextMap()["document"] != "Document (doc-1)" || entries[0].ContextMap()["dataset"] != "Dataset (kb-1)" {
+ t.Errorf("document deletion logs = %v", entries)
+ }
+ if entries := logs.FilterMessage("Deleted dataset").All(); len(entries) != 1 || entries[0].ContextMap()["dataset"] != "Dataset (kb-1)" {
+ t.Errorf("dataset deletion logs = %v", entries)
+ }
+}
+
+func TestDeleteDatasetContinuesWhenStorageIsMissing(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Dataset")
+ if err := db.Exec("INSERT INTO document (id, kb_id, parser_id, parser_config, type, created_by, suffix, location) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", "doc-1", "kb-1", "general", "{}", "pdf", "tenant-1", "pdf", "missing").Error; err != nil {
+ t.Fatal(err)
+ }
+ factory := storage.GetStorageFactory()
+ previous := factory.GetStorage()
+ factory.SetStorage(storage.NewMemoryStorage())
+ t.Cleanup(func() { factory.SetStorage(previous) })
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := testDatasetUpdateService(t).deleteDataset(t.Context(), "tenant-1", kb); err != nil {
+ t.Fatalf("delete dataset with missing storage: %v", err)
+ }
+ var count int64
+ if err := db.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-1").Count(&count).Error; err != nil || count != 0 {
+ t.Fatalf("dataset retained after missing storage: count=%d, err=%v", count, err)
+ }
+}
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index baf0ceddef..13a520b243 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -123,8 +123,15 @@ func (f *fakeUploadStorage) ListObjects(ctx context.Context, bucket string, tena
func (f *fakeUploadStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
return "", nil
}
-func (f *fakeUploadStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
-func (f *fakeUploadStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
+func (f *fakeUploadStorage) BucketExists(ctx context.Context, bucket string) bool { return true }
+func (f *fakeUploadStorage) RemoveBucket(ctx context.Context, bucket string) error { return nil }
+func (f *fakeUploadStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error { return nil }
+func (f *fakeUploadStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
+ return f.ObjExist(ctx, bucket, fnm), nil
+}
+func (f *fakeUploadStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ return true, nil
+}
func (f *fakeUploadStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
v, ok := f.objects[f.key(srcBucket, srcPath)]
if !ok {
diff --git a/internal/service/file/file_delete.go b/internal/service/file/file_delete.go
index 443935faf2..45d4f6dd5a 100644
--- a/internal/service/file/file_delete.go
+++ b/internal/service/file/file_delete.go
@@ -2,12 +2,14 @@ package file
import (
"context"
- "errors"
"fmt"
+
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/storage"
+
+ "go.uber.org/zap"
)
// DeleteFiles deletes files by IDs
@@ -58,17 +60,36 @@ func (s *FileService) DeleteFiles(ctx context.Context, uid string, fileIDs []str
// deleteSingleFile deletes a single file (not folder)
// Matches Python's _delete_single_file function
func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) error {
- // 1. Delete storage object
+ storageImpl := storage.GetStorageFactory().GetStorage()
+ if err := removeFileObject(ctx, storageImpl, file); err != nil {
+ return err
+ }
+ return s.deleteSingleFileRecords(ctx, file)
+}
+
+func removeFileObject(ctx context.Context, storageImpl storage.Storage, file *entity.File) error {
if file.Location != nil && *file.Location != "" {
- storageImpl := storage.GetStorageFactory().GetStorage()
- if storageImpl != nil {
- if err := storageImpl.Remove(ctx, file.ParentID, *file.Location); err != nil {
- common.Logger.Error(fmt.Sprintf("Fail to remove object: %s/%s, error: %v", file.ParentID, *file.Location, err))
- }
+ if storageImpl == nil {
+ return fmt.Errorf("storage is not configured for file %s", file.ID)
+ }
+ exists, err := storageImpl.ObjectExists(ctx, file.ParentID, *file.Location)
+ if err != nil {
+ return fmt.Errorf("check file %s: %w", file.ID, err)
+ }
+ if !exists {
+ common.Warn("File object already missing", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
+ return nil
+ }
+ if err := storageImpl.Remove(ctx, file.ParentID, *file.Location); err != nil {
+ return fmt.Errorf("remove file %s: %w", file.ID, err)
}
+ common.Info("Removed file object", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID), zap.String("bucket", file.ParentID))
}
+ return nil
+}
- // 2. Handle associated documents
+func (s *FileService) deleteSingleFileRecords(ctx context.Context, file *entity.File) error {
+ // Handle associated documents
informs, err := s.file2DocumentDAO.GetByFileID(ctx, dao.DB, file.ID)
if err != nil {
return fmt.Errorf("failed to get file2document mappings: %w", err)
@@ -81,10 +102,7 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
docID := *inform.DocumentID
if s.documentService != nil {
if err = s.documentService.RemoveDocumentKeepFile(ctx, docID); err != nil {
- if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
- return fmt.Errorf("context cancelled while removing document %s: %w", docID, err)
- }
- common.Logger.Error(fmt.Sprintf("Fail to remove document: %s, error: %v", docID, err))
+ return fmt.Errorf("remove document %s: %w", docID, err)
}
}
}
@@ -95,10 +113,11 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
}
}
- // 3. Delete file record
+ // Delete file record
if err = s.fileDAO.Delete(ctx, dao.DB, file.ID); err != nil {
return err
}
+ common.Info("Deleted file", zap.String("file", fileNameAndID(file)), zap.String("file_id", file.ID))
return nil
}
@@ -106,30 +125,72 @@ func (s *FileService) deleteSingleFile(ctx context.Context, file *entity.File) e
// deleteFolderRecursive recursively deletes a folder and its contents
// Matches Python's _delete_folder_recursive function
func (s *FileService) deleteFolderRecursive(ctx context.Context, folder *entity.File, uid string) error {
- // Get all sub-files
+ storageImpl := storage.GetStorageFactory().GetStorage()
+ if storageImpl == nil {
+ return fmt.Errorf("storage is not configured for folder %s", folder.ID)
+ }
+ if err := s.removeFolderObjectsRecursive(ctx, folder, storageImpl); err != nil {
+ return err
+ }
+ return s.deleteFolderRecordsRecursive(ctx, folder)
+}
+
+func (s *FileService) removeFolderObjectsRecursive(ctx context.Context, folder *entity.File, storageImpl storage.Storage) error {
subFiles, err := s.fileDAO.ListByParentID(ctx, dao.DB, folder.ID)
if err != nil {
return err
}
-
for _, subFile := range subFiles {
if subFile.Type == FileTypeFolder {
- // Recursively delete subfolder
- if err = s.deleteFolderRecursive(ctx, subFile, uid); err != nil {
+ if err = s.removeFolderObjectsRecursive(ctx, subFile, storageImpl); err != nil {
return err
}
} else {
- // Delete single file
- if err = s.deleteSingleFile(ctx, subFile); err != nil {
+ if err = removeFileObject(ctx, storageImpl, subFile); err != nil {
return err
}
}
}
+ exists, err := storageImpl.BucketExistsWithError(ctx, folder.ID)
+ if err != nil {
+ return fmt.Errorf("check folder bucket %s: %w", folder.ID, err)
+ }
+ if !exists {
+ common.Warn("Folder bucket already missing", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID))
+ return nil
+ }
+ if err := storageImpl.RemoveEmptyBucket(ctx, folder.ID); err != nil {
+ common.Warn("Unable to remove empty folder bucket", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID), zap.Error(err))
+ } else {
+ common.Info("Removed empty folder bucket", zap.String("folder", fileNameAndID(folder)), zap.String("bucket", folder.ID))
+ }
+ return nil
+}
- // Delete the folder itself
+func (s *FileService) deleteFolderRecordsRecursive(ctx context.Context, folder *entity.File) error {
+ subFiles, err := s.fileDAO.ListByParentID(ctx, dao.DB, folder.ID)
+ if err != nil {
+ return err
+ }
+ for _, subFile := range subFiles {
+ if subFile.Type == FileTypeFolder {
+ if err = s.deleteFolderRecordsRecursive(ctx, subFile); err != nil {
+ return err
+ }
+ } else if err = s.deleteSingleFileRecords(ctx, subFile); err != nil {
+ return err
+ }
+ }
if err = s.fileDAO.Delete(ctx, dao.DB, folder.ID); err != nil {
return err
}
-
+ common.Info("Deleted folder", zap.String("folder", fileNameAndID(folder)), zap.String("folder_id", folder.ID))
return nil
}
+
+func fileNameAndID(file *entity.File) string {
+ if file.Name == "" {
+ return file.ID
+ }
+ return fmt.Sprintf("%s (%s)", file.Name, file.ID)
+}
diff --git a/internal/service/file/file_delete_test.go b/internal/service/file/file_delete_test.go
new file mode 100644
index 0000000000..a1378d5704
--- /dev/null
+++ b/internal/service/file/file_delete_test.go
@@ -0,0 +1,119 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package file
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/storage"
+
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "go.uber.org/zap/zaptest/observer"
+)
+
+type failingObjectCheckStorage struct{ storage.Storage }
+
+func (f failingObjectCheckStorage) ObjectExists(context.Context, string, string) (bool, error) {
+ return false, errors.New("storage unavailable")
+}
+
+func TestDeleteFolderLeavesNonemptyBucket(t *testing.T) {
+ db := setupFolderTestDB(t)
+ folder, err := testFileService().fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
+ if err != nil {
+ t.Fatal(err)
+ }
+ insertFolderTestFile(t, "file-1", folder.ID, "tracked")
+ store := storage.NewMemoryStorage()
+ if err := store.Put(t.Context(), folder.ID, "tracked", []byte("content")); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.Put(t.Context(), folder.ID, "untracked", []byte("content")); err != nil {
+ t.Fatal(err)
+ }
+ factory := storage.GetStorageFactory()
+ previous := factory.GetStorage()
+ factory.SetStorage(store)
+ t.Cleanup(func() { factory.SetStorage(previous) })
+ core, logs := observer.New(zapcore.InfoLevel)
+ previousLogger := common.Logger
+ common.Logger = zap.New(core)
+ t.Cleanup(func() { common.Logger = previousLogger })
+
+ service := testFileService()
+ if err := service.deleteFolderRecursive(t.Context(), folder, "tenant-1"); err != nil {
+ t.Fatalf("delete folder with nonempty bucket: %v", err)
+ }
+ if !store.ObjExist(t.Context(), folder.ID, "untracked") {
+ t.Fatal("folder deletion removed an untracked object")
+ }
+ if store.ObjExist(t.Context(), folder.ID, "tracked") {
+ t.Fatal("folder deletion retained a file object")
+ }
+ if _, err := service.fileDAO.GetByID(t.Context(), db, folder.ID); err == nil {
+ t.Fatal("folder record retained after storage failure")
+ }
+ if entries := logs.FilterMessage("Deleted file").All(); len(entries) != 1 || entries[0].ContextMap()["file"] != "tracked (file-1)" {
+ t.Errorf("file deletion logs = %v", entries)
+ }
+ if entries := logs.FilterMessage("Deleted folder").All(); len(entries) != 1 || entries[0].ContextMap()["folder"] != "folder ("+folder.ID+")" {
+ t.Errorf("folder deletion logs = %v", entries)
+ }
+}
+
+func TestDeleteFolderStopsOnStorageCheckError(t *testing.T) {
+ db := setupFolderTestDB(t)
+ folder, err := testFileService().fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
+ if err != nil {
+ t.Fatal(err)
+ }
+ insertFolderTestFile(t, "file-1", folder.ID, "tracked")
+ factory := storage.GetStorageFactory()
+ previous := factory.GetStorage()
+ factory.SetStorage(failingObjectCheckStorage{storage.NewMemoryStorage()})
+ t.Cleanup(func() { factory.SetStorage(previous) })
+ if err := testFileService().deleteFolderRecursive(t.Context(), folder, "tenant-1"); err == nil {
+ t.Fatal("expected storage check error")
+ }
+ if _, err := testFileService().fileDAO.GetByID(t.Context(), db, "file-1"); err != nil {
+ t.Fatalf("file record deleted after storage check failed: %v", err)
+ }
+}
+
+func TestDeleteFolderContinuesWhenStorageIsMissing(t *testing.T) {
+ db := setupFolderTestDB(t)
+ service := testFileService()
+ folder, err := service.fileDAO.CreateFolder(t.Context(), db, "root", "tenant-1", "folder", FileTypeFolder)
+ if err != nil {
+ t.Fatal(err)
+ }
+ insertFolderTestFile(t, "file-1", folder.ID, "missing")
+ factory := storage.GetStorageFactory()
+ previous := factory.GetStorage()
+ factory.SetStorage(storage.NewMemoryStorage())
+ t.Cleanup(func() { factory.SetStorage(previous) })
+ if err := service.deleteFolderRecursive(t.Context(), folder, "tenant-1"); err != nil {
+ t.Fatalf("delete folder with missing storage: %v", err)
+ }
+ if _, err := service.fileDAO.GetByID(t.Context(), db, folder.ID); err == nil {
+ t.Fatal("folder record retained after missing storage")
+ }
+}
diff --git a/internal/service/file/file_test.go b/internal/service/file/file_test.go
index c79385f384..ab0abcccb7 100644
--- a/internal/service/file/file_test.go
+++ b/internal/service/file/file_test.go
@@ -123,6 +123,16 @@ func (f *fakeStorage) RemoveBucket(ctx context.Context, bucket string) error {
panic("not implemented in fakeStorage")
}
+func (f *fakeStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
+ panic("not implemented in fakeStorage")
+}
+func (f *fakeStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
+ return f.ObjExist(ctx, bucket, fnm), nil
+}
+func (f *fakeStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ panic("not implemented in fakeStorage")
+}
+
func (f *fakeStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
panic("not implemented in fakeStorage")
}
diff --git a/internal/service/memory.go b/internal/service/memory.go
index fdb1409137..cd097fe819 100644
--- a/internal/service/memory.go
+++ b/internal/service/memory.go
@@ -922,7 +922,7 @@ func (s *MemoryService) DeleteMemory(ctx context.Context, userID, memoryID strin
// TODO: Delete associated message index - Implementation pending MessageService
if s.docEngine != nil && engine.IsOceanBaseFamily(s.docEngine.GetType()) {
- if err := s.docEngine.DropChunkStore(ctx, memoryIndexName(memory.TenantID), memoryID); err != nil {
+ if err := s.docEngine.DropChunkStore(ctx, MemoryIndexName(memory.TenantID), memoryID); err != nil {
return fmt.Errorf("delete memory messages: %w", err)
}
}
@@ -964,7 +964,7 @@ func (s *MemoryService) ForgetMessage(ctx context.Context, userID string, memory
condition := map[string]interface{}{
"id": messageDocID,
}
- indexName := memoryIndexName(memory.TenantID)
+ indexName := MemoryIndexName(memory.TenantID)
if err = s.docEngine.UpdateChunks(ctx, condition, updates, indexName, memoryID); err != nil {
if isMessageDocumentNotFound(err) {
@@ -1097,7 +1097,7 @@ func (s *MemoryService) UpdateMessageStatus(ctx context.Context, userID, memoryI
condition := map[string]interface{}{
"id": messageDocID,
}
- indexName := memoryIndexName(memory.TenantID)
+ indexName := MemoryIndexName(memory.TenantID)
if err = s.docEngine.UpdateChunks(ctx, condition, updates, indexName, memoryID); err != nil {
if isMessageDocumentNotFound(err) {
return false, &ResourceNotFoundError{Resource: "Message", ID: messageDocID}
@@ -1121,7 +1121,7 @@ func (s *MemoryService) GetMessageContent(ctx context.Context, userID, memoryID
return nil, errors.New("message store is not initialized")
}
- indexName := memoryIndexName(memory.TenantID)
+ indexName := MemoryIndexName(memory.TenantID)
docID := fmt.Sprintf("%s_%d", memoryID, messageID)
res, err := s.docEngine.GetChunk(ctx, indexName, docID, []string{memoryID})
if err != nil {
@@ -1423,7 +1423,8 @@ func memoryMessageSelectFields() []string {
}
}
-func memoryIndexName(tenantID string) string {
+// MemoryIndexName returns the message index shared by a tenant's memories.
+func MemoryIndexName(tenantID string) string {
prefix := strings.TrimSpace(common.GetEnv(common.EnvESIndexPrefix))
if prefix == "" {
return fmt.Sprintf("memory_%s", tenantID)
@@ -1438,7 +1439,7 @@ func memorySearchIndexNames(memories []*entity.Memory) []string {
if memory == nil {
continue
}
- indexName := memoryIndexName(memory.TenantID)
+ indexName := MemoryIndexName(memory.TenantID)
if engine.GetEngineType() == "infinity" {
indexName = fmt.Sprintf("%s_%s", indexName, memory.ID)
}
diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.go
index cd00eae65d..a1f2463253 100644
--- a/internal/service/memory_message_service.go
+++ b/internal/service/memory_message_service.go
@@ -353,7 +353,7 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr
message["doc_id"] = message["memory_id"]
}
- indexName := memoryIndexName(mem.TenantID)
+ indexName := MemoryIndexName(mem.TenantID)
exists, err := s.memories.docEngine.ChunkStoreExists(ctx, indexName, mem.ID)
if err != nil {
return fmt.Errorf("check message index: %w", err)
diff --git a/internal/service/memory_message_test.go b/internal/service/memory_message_test.go
index 220b152ddd..b5287fba55 100644
--- a/internal/service/memory_message_test.go
+++ b/internal/service/memory_message_test.go
@@ -34,12 +34,12 @@ func TestIsMessageDocumentNotFound(t *testing.T) {
func TestMemoryIndexNameMatchesPythonPrefix(t *testing.T) {
t.Setenv(common.EnvESIndexPrefix, "")
- if got := memoryIndexName("tenant-1"); got != "memory_tenant-1" {
- t.Fatalf("memoryIndexName() = %q", got)
+ if got := MemoryIndexName("tenant-1"); got != "memory_tenant-1" {
+ t.Fatalf("MemoryIndexName() = %q", got)
}
t.Setenv(common.EnvESIndexPrefix, "legacy")
- if got := memoryIndexName("tenant-1"); got != "memory_legacy_tenant-1" {
- t.Fatalf("memoryIndexName() with prefix = %q", got)
+ if got := MemoryIndexName("tenant-1"); got != "memory_legacy_tenant-1" {
+ t.Fatalf("MemoryIndexName() with prefix = %q", got)
}
}
diff --git a/internal/service/skill_indexer.go b/internal/service/skill_indexer.go
index 0937785d37..2d20172732 100644
--- a/internal/service/skill_indexer.go
+++ b/internal/service/skill_indexer.go
@@ -168,7 +168,7 @@ func (s *SkillIndexerService) IndexSkill(ctx context.Context, tenantID, spaceID
}
}
- indexName := getSkillIndexName(tenantID, spaceID)
+ indexName := SkillIndexName(tenantID, spaceID)
// For Infinity: ensure table exists with correct dimension BEFORE inserting
if docEngine.GetType() == "infinity" {
@@ -249,7 +249,7 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp
}
// Ensure index exists with correct dimension
- indexName := getSkillIndexName(tenantID, spaceID)
+ indexName := SkillIndexName(tenantID, spaceID)
if docEngine.GetType() == "infinity" {
// For Infinity: must ensure table exists with correct dimension BEFORE inserting
common.Info(fmt.Sprintf("Checking if index exists: %s", indexName))
@@ -362,7 +362,7 @@ func (s *SkillIndexerService) BatchIndexSkills(ctx context.Context, tenantID, sp
// Returns nil if the document doesn't exist (idempotent delete)
func (s *SkillIndexerService) DeleteSkillIndex(ctx context.Context, tenantID, spaceID, skillID string, docEngine engine.DocEngine) error {
spaceID = normalizeSpaceID(spaceID)
- indexName := getSkillIndexName(tenantID, spaceID)
+ indexName := SkillIndexName(tenantID, spaceID)
// ES document ID cannot contain '/' - replace with '_'
docID := strings.ReplaceAll(skillID, "/", "_")
if err := docEngine.DeleteDocument(ctx, indexName, docID); err != nil {
@@ -381,7 +381,7 @@ func (s *SkillIndexerService) DeleteSkillIndex(ctx context.Context, tenantID, sp
// Deletes all versions: both new format (skillname) and old format (skillname_x.x.x)
func (s *SkillIndexerService) DeleteSkillByName(ctx context.Context, tenantID, spaceID, skillName string, docEngine engine.DocEngine) error {
spaceID = normalizeSpaceID(spaceID)
- indexName := getSkillIndexName(tenantID, spaceID)
+ indexName := SkillIndexName(tenantID, spaceID)
docID := strings.ReplaceAll(skillName, "/", "_")
if err := docEngine.DeleteDocument(ctx, indexName, docID); err != nil {
@@ -436,7 +436,7 @@ func (s *SkillIndexerService) ReindexAll(ctx context.Context, tenantID, spaceID
common.Info(fmt.Sprintf("ReindexAll: new embedding dimension is %d", newDimension))
// Delete existing index and recreate with new dimension (for both ES and Infinity)
- indexName := getSkillIndexName(tenantID, spaceID)
+ indexName := SkillIndexName(tenantID, spaceID)
exists, _ := docEngine.ChunkStoreExists(ctx, indexName, "skill")
if exists {
common.Info(fmt.Sprintf("ReindexAll: deleting existing index %s", indexName))
@@ -843,7 +843,7 @@ func (s *SkillIndexerService) cleanupOldVersions(ctx context.Context, tenantID,
// InitializeIndex initializes the skill search index for a tenant
func (s *SkillIndexerService) InitializeIndex(ctx context.Context, tenantID, spaceID string, docEngine engine.DocEngine, embdID string) error {
// Check if index exists
- indexName := getSkillIndexName(tenantID, spaceID)
+ indexName := SkillIndexName(tenantID, spaceID)
common.Info("Checking skill index existence", zap.String("indexName", indexName), zap.String("tenantID", tenantID), zap.String("spaceID", spaceID))
@@ -874,7 +874,7 @@ func (s *SkillIndexerService) createIndex(ctx context.Context, tenantID, spaceID
// createIndexWithDimension creates the skill index with a specific vector dimension
func (s *SkillIndexerService) createIndexWithDimension(ctx context.Context, tenantID, spaceID string, docEngine engine.DocEngine, embdID string, dimension int) error {
- indexName := getSkillIndexName(tenantID, spaceID)
+ indexName := SkillIndexName(tenantID, spaceID)
common.Info(fmt.Sprintf("Creating skill index with dimension %d", dimension),
zap.String("indexName", indexName),
diff --git a/internal/service/skill_search.go b/internal/service/skill_search.go
index 2793a277f5..b380342f1f 100644
--- a/internal/service/skill_search.go
+++ b/internal/service/skill_search.go
@@ -222,7 +222,7 @@ func (s *SkillSearchService) Search(ctx context.Context, req *SearchRequest, doc
}
// Check if index exists before searching
- indexName := getSkillIndexName(req.TenantID, req.SpaceID)
+ indexName := SkillIndexName(req.TenantID, req.SpaceID)
common.Debug("Searching skills", zap.String("indexName", indexName), zap.String("query", req.Query))
indexExists, err := docEngine.ChunkStoreExists(ctx, indexName, "skill")
@@ -694,7 +694,8 @@ func (s *SkillSearchService) getEmbedding(ctx context.Context, text, embdID, ten
}
// Helper functions
-func getSkillIndexName(tenantID, spaceID string) string {
+// SkillIndexName returns the index used by a tenant's skill space.
+func SkillIndexName(tenantID, spaceID string) string {
spaceID = normalizeSpaceID(spaceID)
spaceID = strings.ToLower(spaceID)
replacer := strings.NewReplacer("-", "_", "/", "_", "\\", "_", " ", "_", ".", "_", ":", "_")
diff --git a/internal/service/skill_space.go b/internal/service/skill_space.go
index c69ed0133a..f3f9f485fc 100644
--- a/internal/service/skill_space.go
+++ b/internal/service/skill_space.go
@@ -439,7 +439,7 @@ func (s *SkillSpaceService) asyncDeleteSpace(ctx context.Context, spaceID, folde
// Step 1: Delete the search index
if docEngine != nil {
- indexName := getSkillIndexName(tenantID, spaceID)
+ indexName := SkillIndexName(tenantID, spaceID)
common.Info("Async deleting space index", zap.String("index", indexName), zap.String("spaceID", spaceID))
deleteCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
if err := docEngine.DropChunkStore(deleteCtx, indexName, "skill"); err != nil {
diff --git a/internal/storage/gcs.go b/internal/storage/gcs.go
index 191c7906df..78de6a224e 100644
--- a/internal/storage/gcs.go
+++ b/internal/storage/gcs.go
@@ -111,6 +111,9 @@ func (m *GCSStorage) Remove(ctx context.Context, bucketName, objectName string,
obj := m.client.Bucket(bucketName).Object(objectName)
if err := obj.Delete(ctx); err != nil {
+ if errors.Is(err, storage.ErrObjectNotExist) || errors.Is(err, storage.ErrBucketNotExist) {
+ return nil
+ }
return fmt.Errorf("fail to delete object: %w", err)
}
@@ -130,6 +133,17 @@ func (m *GCSStorage) ObjExist(ctx context.Context, bucketName, objectName string
return true
}
+func (m *GCSStorage) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, error) {
+ _, err := m.client.Bucket(bucketName).Object(objectName).Attrs(ctx)
+ if err == nil {
+ return true, nil
+ }
+ if errors.Is(err, storage.ErrObjectNotExist) || errors.Is(err, storage.ErrBucketNotExist) {
+ return false, nil
+ }
+ return false, err
+}
+
func (m *GCSStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
bucketObject := m.client.Bucket(bucket)
@@ -182,6 +196,17 @@ func (m *GCSStorage) BucketExists(ctx context.Context, bucket string) bool {
return true
}
+func (m *GCSStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ _, err := m.client.Bucket(bucket).Attrs(ctx)
+ if err == nil {
+ return true, nil
+ }
+ if errors.Is(err, storage.ErrBucketNotExist) {
+ return false, nil
+ }
+ return false, err
+}
+
// RemoveBucket removes a bucket and all its objects
func (m *GCSStorage) RemoveBucket(ctx context.Context, bucketName string) error {
if bucketName == "" {
@@ -212,6 +237,18 @@ func (m *GCSStorage) RemoveBucket(ctx context.Context, bucketName string) error
return nil
}
+// RemoveEmptyBucket removes a bucket only if GCS finds it empty.
+func (m *GCSStorage) RemoveEmptyBucket(ctx context.Context, bucketName string) error {
+ if bucketName == "" {
+ return fmt.Errorf("attempt to delete bucket without name")
+ }
+ err := m.client.Bucket(bucketName).Delete(ctx)
+ if errors.Is(err, storage.ErrBucketNotExist) {
+ return nil
+ }
+ return err
+}
+
// Copy copies an object from source to destination
func (m *GCSStorage) Copy(ctx context.Context, srcBucket, srcObject, destBucket, destObject string) bool {
diff --git a/internal/storage/memory.go b/internal/storage/memory.go
index a67f6143a2..10d3edc8aa 100644
--- a/internal/storage/memory.go
+++ b/internal/storage/memory.go
@@ -137,6 +137,16 @@ func (m *MemoryStorage) ObjExist(ctx context.Context, bucket, fnm string, tenant
return ok
}
+func (m *MemoryStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
+ if err := ctx.Err(); err != nil {
+ return false, err
+ }
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ _, exists := m.objects[bucket][fnm]
+ return exists, nil
+}
+
func (m *MemoryStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
m.mu.RLock()
defer m.mu.RUnlock()
@@ -183,6 +193,13 @@ func (m *MemoryStorage) BucketExists(ctx context.Context, bucket string) bool {
return ok
}
+func (m *MemoryStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ if err := ctx.Err(); err != nil {
+ return false, err
+ }
+ return m.BucketExists(ctx, bucket), nil
+}
+
// RemoveBucket deletes a bucket and all of its keys. Removing a
// non-existent bucket is a no-op and returns nil.
func (m *MemoryStorage) RemoveBucket(ctx context.Context, bucket string) error {
@@ -197,6 +214,20 @@ func (m *MemoryStorage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
+// RemoveEmptyBucket deletes a bucket only when it contains no objects.
+func (m *MemoryStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if len(m.objects[bucket]) != 0 {
+ return fmt.Errorf("bucket %s is not empty", bucket)
+ }
+ delete(m.objects, bucket)
+ return nil
+}
+
// Copy duplicates an object from srcBucket/srcKey to destBucket/destKey.
// The source is left untouched. Returns false if the source does not exist
// or if the destination bucket creation fails.
diff --git a/internal/storage/memory_test.go b/internal/storage/memory_test.go
index 2c20ad09e0..a8e596a8d8 100644
--- a/internal/storage/memory_test.go
+++ b/internal/storage/memory_test.go
@@ -156,6 +156,29 @@ func TestMemoryStorage_RemoveBucket(t *testing.T) {
}
}
+func TestMemoryStorage_RemoveEmptyBucket(t *testing.T) {
+ ms := newTestMemory(t)
+ ctx := t.Context()
+ if err := ms.Put(ctx, "bucket", "file", []byte("content")); err != nil {
+ t.Fatal(err)
+ }
+ if err := ms.RemoveEmptyBucket(ctx, "bucket"); err == nil {
+ t.Fatal("RemoveEmptyBucket removed a nonempty bucket")
+ }
+ if !ms.ObjExist(ctx, "bucket", "file") {
+ t.Fatal("RemoveEmptyBucket removed the file")
+ }
+ if err := ms.Remove(ctx, "bucket", "file"); err != nil {
+ t.Fatal(err)
+ }
+ if err := ms.RemoveEmptyBucket(ctx, "bucket"); err != nil {
+ t.Fatal(err)
+ }
+ if ms.BucketExists(ctx, "bucket") {
+ t.Fatal("RemoveEmptyBucket retained an empty bucket")
+ }
+}
+
func TestMemoryStorage_CopyMove(t *testing.T) {
ms := newTestMemory(t)
ctx := t.Context()
diff --git a/internal/storage/minio.go b/internal/storage/minio.go
index df5342c6d5..1ae6615b14 100644
--- a/internal/storage/minio.go
+++ b/internal/storage/minio.go
@@ -234,6 +234,10 @@ func (m *MinioStorage) Remove(ctx context.Context, bucket, fnm string, tenantID
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
if err := m.client.RemoveObject(ctx, bucket, fnm, minio.RemoveObjectOptions{}); err != nil {
+ code := minio.ToErrorResponse(err).Code
+ if code == "NoSuchKey" || code == "NoSuchBucket" {
+ return nil
+ }
common.Warn("Failed to remove object", zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
return err
}
@@ -263,6 +267,19 @@ func (m *MinioStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantI
return true
}
+func (m *MinioStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
+ bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
+ _, err := m.client.StatObject(ctx, bucket, fnm, minio.StatObjectOptions{})
+ if err == nil {
+ return true, nil
+ }
+ code := minio.ToErrorResponse(err).Code
+ if code == "NoSuchKey" || code == "NoSuchBucket" {
+ return false, nil
+ }
+ return false, err
+}
+
// GetPresignedURL generates a presigned URL for accessing an object
func (m *MinioStorage) GetPresignedURL(ctx context.Context, bucket, fnm string, expires time.Duration, tenantID ...string) (string, error) {
bucket, fnm = m.resolveBucketAndPath(bucket, fnm)
@@ -303,6 +320,13 @@ func (m *MinioStorage) BucketExists(ctx context.Context, bucket string) bool {
return exists
}
+func (m *MinioStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ if m.bucket != "" {
+ bucket = m.bucket
+ }
+ return m.client.BucketExists(ctx, bucket)
+}
+
func (m *MinioStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
var objects []string
@@ -401,6 +425,41 @@ func (m *MinioStorage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
+// RemoveEmptyBucket removes a bucket only when it contains no object versions.
+func (m *MinioStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
+ actualBucket := bucket
+ prefix := ""
+ if m.bucket != "" {
+ actualBucket = m.bucket
+ if m.prefixPath != "" {
+ prefix = m.prefixPath + "/"
+ }
+ prefix += bucket + "/"
+ }
+ exists, err := m.client.BucketExists(ctx, actualBucket)
+ if err != nil {
+ return err
+ }
+ if !exists {
+ return nil
+ }
+ listCtx, cancel := context.WithCancel(ctx)
+ defer cancel()
+ for object := range m.client.ListObjects(listCtx, actualBucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true, WithVersions: true}) {
+ if object.Err != nil {
+ return object.Err
+ }
+ return fmt.Errorf("bucket %s is not empty", bucket)
+ }
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if m.bucket == "" {
+ return m.client.RemoveBucket(ctx, actualBucket)
+ }
+ return nil
+}
+
// Copy copies an object from source to destination
func (m *MinioStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
srcBucket, srcPath = m.resolveBucketAndPath(srcBucket, srcPath)
diff --git a/internal/storage/oss.go b/internal/storage/oss.go
index 05e88bf7d7..ba7e951741 100644
--- a/internal/storage/oss.go
+++ b/internal/storage/oss.go
@@ -253,6 +253,9 @@ func (o *OSSStorage) Remove(ctx context.Context, bucket, fnm string, tenantID ..
Key: aws.String(fnm),
})
if err != nil {
+ if isS3NotFound(err) {
+ return nil
+ }
common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm))
return err
}
@@ -278,6 +281,18 @@ func (o *OSSStorage) ObjExist(ctx context.Context, bucket, fnm string, tenantID
return true
}
+func (o *OSSStorage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
+ bucket, fnm = o.resolveBucketAndPath(bucket, fnm)
+ _, err := o.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})
+ if err == nil {
+ return true, nil
+ }
+ if isS3NotFound(err) {
+ return false, nil
+ }
+ return false, err
+}
+
func (o *OSSStorage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
listInput := &s3.ListObjectsV2Input{
@@ -344,13 +359,28 @@ func (o *OSSStorage) BucketExists(ctx context.Context, bucket string) bool {
return true
}
+func (o *OSSStorage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ if o.bucket != "" {
+ bucket = o.bucket
+ }
+ _, err := o.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
+ if err == nil {
+ return true, nil
+ }
+ if isS3BucketNotFound(err) {
+ return false, nil
+ }
+ return false, err
+}
+
// RemoveBucket removes a bucket and all its objects
func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
- actualBucket := bucket
if o.bucket != "" {
- actualBucket = o.bucket
+ return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)
}
+ actualBucket := bucket
+
// Check if bucket exists
if !o.BucketExists(ctx, actualBucket) {
return nil
@@ -396,6 +426,18 @@ func (o *OSSStorage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
+// RemoveEmptyBucket removes a physical bucket only when it is empty.
+func (o *OSSStorage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
+ if o.bucket != "" {
+ return fmt.Errorf("cannot remove logical OSS bucket %s: shared bucket mode does not isolate objects by logical bucket", bucket)
+ }
+ _, err := o.client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(bucket)})
+ if isS3NotFound(err) {
+ return nil
+ }
+ return err
+}
+
// Copy copies an object from source to destination
func (o *OSSStorage) Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool {
srcBucket, srcPath = o.resolveBucketAndPath(srcBucket, srcPath)
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index be3522adfc..15c6957b72 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -262,6 +262,9 @@ func (s *S3Storage) Remove(ctx context.Context, bucket, fnm string, tenantID ...
Key: aws.String(fnm),
})
if err != nil {
+ if isS3NotFound(err) {
+ return nil
+ }
common.Error("Failed to remove object", err, zap.String("bucket", bucket), zap.String("key", fnm), zap.Error(err))
return err
}
@@ -287,6 +290,18 @@ func (s *S3Storage) ObjExist(ctx context.Context, bucket, fnm string, tenantID .
return true
}
+func (s *S3Storage) ObjectExists(ctx context.Context, bucket, fnm string) (bool, error) {
+ bucket, fnm = s.resolveBucketAndPath(bucket, fnm)
+ _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String(bucket), Key: aws.String(fnm)})
+ if err == nil {
+ return true, nil
+ }
+ if isS3NotFound(err) {
+ return false, nil
+ }
+ return false, err
+}
+
func (s *S3Storage) ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error) {
listInput := &s3.ListObjectsV2Input{
@@ -349,6 +364,11 @@ func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {
return true
}
+func (s *S3Storage) BucketExistsWithError(ctx context.Context, bucket string) (bool, error) {
+ actualBucket, _ := s.resolveBucketAndPrefix(bucket)
+ return s.bucketExistsForRemoval(ctx, actualBucket)
+}
+
// RemoveBucket removes a bucket and all its objects
func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
actualBucket, prefix := s.resolveBucketAndPrefix(bucket)
@@ -378,6 +398,29 @@ func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
+// RemoveEmptyBucket removes a bucket only when it contains no object versions.
+func (s *S3Storage) RemoveEmptyBucket(ctx context.Context, bucket string) error {
+ actualBucket, prefix := s.resolveBucketAndPrefix(bucket)
+ exists, err := s.bucketExistsForRemoval(ctx, actualBucket)
+ if err != nil || !exists {
+ return err
+ }
+ if s.bucket != "" {
+ versions, err := s.client.ListObjectVersions(ctx, &s3.ListObjectVersionsInput{
+ Bucket: aws.String(actualBucket), Prefix: aws.String(prefix), MaxKeys: aws.Int32(1),
+ })
+ if err != nil {
+ return err
+ }
+ if len(versions.Versions) > 0 || len(versions.DeleteMarkers) > 0 || (versions.IsTruncated != nil && *versions.IsTruncated) {
+ return fmt.Errorf("bucket %s is not empty", bucket)
+ }
+ return nil
+ }
+ _, err = s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(actualBucket)})
+ return err
+}
+
func (s *S3Storage) resolveBucketAndPrefix(bucket string) (string, string) {
if s.bucket == "" {
return bucket, ""
@@ -490,7 +533,7 @@ func isS3NotFound(err error) bool {
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
- return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey"
+ return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchKey" || apiErr.ErrorCode() == "NoSuchBucket"
}
return false
}
diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go
index f643791094..621a0e5f85 100644
--- a/internal/storage/s3_test.go
+++ b/internal/storage/s3_test.go
@@ -129,6 +129,93 @@ func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {
}
}
+func TestS3StorageRemoveEmptyBucketPreservesNonemptyPhysicalBucket(t *testing.T) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
+ switch req.Method {
+ case http.MethodHead:
+ return s3Response(http.StatusOK, ""), nil
+ case http.MethodDelete:
+ return s3Response(http.StatusConflict, `<Error><Code>BucketNotEmpty</Code></Error>`), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err == nil {
+ t.Fatal("RemoveEmptyBucket returned nil for a nonempty bucket")
+ }
+}
+
+func TestS3StorageExistenceChecksDistinguishMissingFromFailure(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ status int
+ code string
+ missing bool
+ }{
+ {name: "missing", status: http.StatusNotFound, code: "NoSuchBucket", missing: true},
+ {name: "unavailable", status: http.StatusInternalServerError, code: "InternalError"},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
+ return s3Response(test.status, `<Error><Code>`+test.code+`</Code></Error>`), nil
+ })
+ objectExists, objectErr := storage.ObjectExists(t.Context(), "kb01", "file")
+ bucketExists, bucketErr := storage.BucketExistsWithError(t.Context(), "kb01")
+ if objectExists || bucketExists {
+ t.Fatal("existence check reported an existing item")
+ }
+ if test.missing && (objectErr != nil || bucketErr != nil) {
+ t.Fatalf("missing item returned errors: object=%v bucket=%v", objectErr, bucketErr)
+ }
+ if !test.missing && (objectErr == nil || bucketErr == nil) {
+ t.Fatalf("storage failure was treated as missing: object=%v bucket=%v", objectErr, bucketErr)
+ }
+ })
+ }
+}
+
+func TestS3StorageRemoveEmptyBucketDeletesEmptyPhysicalBucket(t *testing.T) {
+ deleted := false
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
+ switch req.Method {
+ case http.MethodHead:
+ return s3Response(http.StatusOK, ""), nil
+ case http.MethodDelete:
+ deleted = true
+ return s3Response(http.StatusNoContent, ""), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err != nil {
+ t.Fatal(err)
+ }
+ if !deleted {
+ t.Fatal("RemoveEmptyBucket did not delete an empty bucket")
+ }
+}
+
+func TestS3StorageRemoveEmptyBucketPreservesLogicalBucketObjects(t *testing.T) {
+ storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead:
+ return s3Response(http.StatusOK, ""), nil
+ case req.URL.Query().Has("versions"):
+ if got := req.URL.Query().Get("prefix"); got != "prefix/kb01/" {
+ t.Fatalf("prefix = %q", got)
+ }
+ return s3Response(http.StatusOK, `<ListVersionsResult><Version><Key>prefix/kb01/file</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ storage.bucket = "physical"
+ storage.prefixPath = "prefix"
+ if err := storage.RemoveEmptyBucket(t.Context(), "kb01"); err == nil {
+ t.Fatal("RemoveEmptyBucket returned nil for a nonempty logical bucket")
+ }
+}
+
func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {
var deleteBody string
storage := newS3TestStorage("us-east-1", func(req *http.Request) (*http.Response, error) {
diff --git a/internal/storage/types.go b/internal/storage/types.go
index d9d6857ae0..663fd7035c 100644
--- a/internal/storage/types.go
+++ b/internal/storage/types.go
@@ -78,6 +78,8 @@ type Storage interface {
// ObjExist checks if an object exists
ObjExist(ctx context.Context, bucket, fnm string, tenantID ...string) bool
+ // ObjectExists distinguishes a missing object from a storage error
+ ObjectExists(ctx context.Context, bucket, fnm string) (bool, error)
// ListObjects list all objects of the bucket
ListObjects(ctx context.Context, bucket string, tenantID ...string) ([]string, error)
@@ -88,10 +90,15 @@ type Storage interface {
// BucketExists checks if a bucket exists
BucketExists(ctx context.Context, bucket string) bool
+ // BucketExistsWithError distinguishes a missing bucket from a storage error
+ BucketExistsWithError(ctx context.Context, bucket string) (bool, error)
// RemoveBucket removes a bucket and all its objects
RemoveBucket(ctx context.Context, bucket string) error
+ // RemoveEmptyBucket removes a bucket only if it has no objects
+ RemoveEmptyBucket(ctx context.Context, bucket string) error
+
// Copy copies an object from source to destination
Copy(ctx context.Context, srcBucket, srcPath, destBucket, destPath string) bool