Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(storage): handle versioned S3 bucket deletion (#19199)

Sbaaoui Idriss9f73625f532d
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index 78f4c376e3..60bd06a3cf 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -29,6 +29,7 @@ import (
s3Config "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"go.uber.org/zap"
)
@@ -334,53 +335,79 @@ func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {
// RemoveBucket removes a bucket and all its objects
func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
- actualBucket := bucket
- if s.bucket != "" {
- actualBucket = s.bucket
+ exists, err := s.bucketExistsForRemoval(ctx, bucket)
+ if err != nil {
+ return err
}
-
- // Check if bucket exists
- if !s.BucketExists(ctx, actualBucket) {
+ if !exists {
return nil
}
- // List and delete all objects
- listInput := &s3.ListObjectsV2Input{
- Bucket: aws.String(actualBucket),
+ if err := s.removeObjects(ctx, bucket); err != nil {
+ return err
}
- for {
- result, err := s.client.ListObjectsV2(ctx, listInput)
- if err != nil {
- common.Error("Failed to list objects", err, zap.String("bucket", actualBucket), zap.Error(err))
- return err
- }
+ _, err = s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+ Bucket: aws.String(bucket),
+ })
+ if err != nil {
+ common.Error("Failed to delete bucket", err, zap.String("bucket", bucket), zap.Error(err))
+ return err
+ }
- for _, obj := range result.Contents {
- _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
- Bucket: aws.String(actualBucket),
- Key: obj.Key,
- })
- if err != nil {
- common.Error("Failed to delete object", err, zap.String("bucket", actualBucket), zap.Error(err))
- }
- }
+ return nil
+}
- if result.IsTruncated == nil || !*result.IsTruncated {
- break
- }
- listInput.ContinuationToken = result.NextContinuationToken
+func (s *S3Storage) bucketExistsForRemoval(ctx context.Context, bucket string) (bool, error) {
+ _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
+ if err == nil {
+ return true, nil
+ }
+ if isS3BucketNotFound(err) {
+ return false, nil
}
+ return false, fmt.Errorf("head S3 bucket %q: %w", bucket, err)
+}
- // Delete bucket
- _, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
- Bucket: aws.String(actualBucket),
+func (s *S3Storage) removeObjects(ctx context.Context, bucket string) error {
+ versions := s3.NewListObjectVersionsPaginator(s.client, &s3.ListObjectVersionsInput{
+ Bucket: aws.String(bucket),
})
- if err != nil {
- common.Error("Failed to delete bucket", err, zap.String("bucket", actualBucket), zap.Error(err))
- return err
+ for versions.HasMorePages() {
+ page, err := versions.NextPage(ctx)
+ if err != nil {
+ return fmt.Errorf("list S3 object versions in %q: %w", bucket, err)
+ }
+ toDelete := make([]types.ObjectIdentifier, 0, len(page.Versions)+len(page.DeleteMarkers))
+ for _, version := range page.Versions {
+ toDelete = append(toDelete, types.ObjectIdentifier{Key: version.Key, VersionId: version.VersionId})
+ }
+ for _, marker := range page.DeleteMarkers {
+ toDelete = append(toDelete, types.ObjectIdentifier{Key: marker.Key, VersionId: marker.VersionId})
+ }
+ if err := s.deleteObjects(ctx, bucket, toDelete); err != nil {
+ return err
+ }
}
+ return nil
+}
+func (s *S3Storage) deleteObjects(ctx context.Context, bucket string, objects []types.ObjectIdentifier) error {
+ for len(objects) > 0 {
+ count := min(len(objects), 1000)
+ output, err := s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
+ Bucket: aws.String(bucket),
+ Delete: &types.Delete{Objects: objects[:count], Quiet: aws.Bool(true)},
+ })
+ if err != nil {
+ return fmt.Errorf("delete S3 objects in %q: %w", bucket, err)
+ }
+ if len(output.Errors) > 0 {
+ failed := output.Errors[0]
+ return fmt.Errorf("delete S3 object %q (version %q): %s: %s", aws.ToString(failed.Key), aws.ToString(failed.VersionId), aws.ToString(failed.Code), aws.ToString(failed.Message))
+ }
+ objects = objects[count:]
+ }
return nil
}
@@ -435,3 +462,14 @@ func isS3NotFound(err error) bool {
}
return false
}
+
+func isS3BucketNotFound(err error) bool {
+ if err == nil {
+ return false
+ }
+ var apiErr smithy.APIError
+ if errors.As(err, &apiErr) {
+ return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchBucket"
+ }
+ return false
+}
diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go
new file mode 100644
index 0000000000..113f947af9
--- /dev/null
+++ b/internal/storage/s3_test.go
@@ -0,0 +1,234 @@
+package storage
+
+import (
+ "io"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/aws/aws-sdk-go-v2/service/s3/types"
+)
+
+type s3RoundTripper func(*http.Request) (*http.Response, error)
+
+func (f s3RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }
+
+func newS3TestStorage(handler s3RoundTripper) *S3Storage {
+ client := s3.NewFromConfig(aws.Config{
+ Region: "us-east-1",
+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),
+ HTTPClient: &http.Client{Transport: handler},
+ }, func(options *s3.Options) {
+ options.BaseEndpoint = aws.String("https://s3.test")
+ options.UsePathStyle = true
+ options.RetryMaxAttempts = 1
+ })
+ return &S3Storage{client: client}
+}
+
+func s3Response(status int, body string) *http.Response {
+ return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}
+}
+
+func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {
+ bucketDeleted := false
+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead:
+ return s3Response(http.StatusOK, ""), nil
+ case req.URL.Query().Has("versions"):
+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated></ListVersionsResult>`), nil
+ case req.Method == http.MethodDelete:
+ if req.URL.Path != "/kb01" {
+ t.Fatalf("DeleteBucket path = %q, want /kb01", req.URL.Path)
+ }
+ bucketDeleted = true
+ return s3Response(http.StatusNoContent, ""), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {
+ t.Fatal(err)
+ }
+ if !bucketDeleted {
+ t.Fatal("RemoveBucket() did not delete the empty physical bucket")
+ }
+}
+
+func TestS3StorageRemoveBucketDeletesVersionsAndMarkers(t *testing.T) {
+ var deleteBody string
+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead:
+ return s3Response(http.StatusOK, ""), nil
+ case req.URL.Query().Has("versions"):
+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><Version><Key>current</Key><VersionId>v2</VersionId></Version><Version><Key>old</Key><VersionId>v1</VersionId></Version><DeleteMarker><Key>removed</Key><VersionId>m1</VersionId></DeleteMarker></ListVersionsResult>`), nil
+ case req.URL.Query().Has("delete"):
+ body, _ := io.ReadAll(req.Body)
+ deleteBody = string(body)
+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil
+ case req.Method == http.MethodDelete:
+ return s3Response(http.StatusNoContent, ""), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {
+ t.Fatal(err)
+ }
+ for _, value := range []string{"<Key>current</Key><VersionId>v2</VersionId>", "<Key>old</Key><VersionId>v1</VersionId>", "<Key>removed</Key><VersionId>m1</VersionId>"} {
+ if !strings.Contains(deleteBody, value) {
+ t.Fatalf("DeleteObjects did not preserve %q: %s", value, deleteBody)
+ }
+ }
+}
+
+func TestS3StorageRemoveBucketDeletesNullVersion(t *testing.T) {
+ var deleteBody string
+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead:
+ return s3Response(http.StatusOK, ""), nil
+ case req.URL.Query().Has("versions"):
+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><Version><Key>document</Key><VersionId>null</VersionId></Version></ListVersionsResult>`), nil
+ case req.URL.Query().Has("delete"):
+ body, _ := io.ReadAll(req.Body)
+ deleteBody = string(body)
+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil
+ case req.Method == http.MethodDelete:
+ return s3Response(http.StatusNoContent, ""), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(deleteBody, `<Key>document</Key><VersionId>null</VersionId>`) {
+ t.Fatalf("DeleteObjects omitted the null version ID: %s", deleteBody)
+ }
+}
+
+func TestS3StorageRemoveBucketVersionPagination(t *testing.T) {
+ versionCalls := 0
+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead:
+ return s3Response(http.StatusOK, ""), nil
+ case req.URL.Query().Has("versions"):
+ versionCalls++
+ if versionCalls == 1 {
+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>true</IsTruncated><NextKeyMarker>a</NextKeyMarker><NextVersionIdMarker>v1</NextVersionIdMarker><Version><Key>a</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil
+ }
+ if req.URL.Query().Get("key-marker") != "a" || req.URL.Query().Get("version-id-marker") != "v1" {
+ t.Fatalf("missing version continuation markers: %s", req.URL)
+ }
+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><DeleteMarker><Key>b</Key><VersionId>m1</VersionId></DeleteMarker></ListVersionsResult>`), nil
+ case req.URL.Query().Has("delete"):
+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil
+ case req.Method == http.MethodDelete:
+ return s3Response(http.StatusNoContent, ""), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {
+ t.Fatal(err)
+ }
+ if versionCalls != 2 {
+ t.Fatalf("ListObjectVersions calls = %d, want 2", versionCalls)
+ }
+}
+
+func TestS3StorageRemoveBucketFailures(t *testing.T) {
+ t.Run("batch limit", func(t *testing.T) {
+ requests := 0
+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ if !req.URL.Query().Has("delete") {
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ }
+ requests++
+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil
+ })
+ objects := make([]types.ObjectIdentifier, 1001)
+ for i := range objects {
+ objects[i].Key = aws.String("key")
+ }
+ if err := storage.deleteObjects(t.Context(), "kb01", objects); err != nil {
+ t.Fatal(err)
+ }
+ if requests != 2 {
+ t.Fatalf("DeleteObjects requests = %d, want 2", requests)
+ }
+ })
+
+ for _, test := range []struct {
+ name, body string
+ status int
+ }{
+ {"list failure", `<Error><Code>InternalError</Code></Error>`, http.StatusInternalServerError},
+ {"embedded delete error", `<DeleteResult><Error><Key>a</Key><Code>AccessDenied</Code><Message>denied</Message></Error></DeleteResult>`, http.StatusOK},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ bucketDeleted := false
+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead:
+ return s3Response(http.StatusOK, ""), nil
+ case req.URL.Query().Has("versions"):
+ if test.name == "list failure" {
+ return s3Response(test.status, test.body), nil
+ }
+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><Version><Key>a</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil
+ case req.URL.Query().Has("delete"):
+ return s3Response(test.status, test.body), nil
+ case req.Method == http.MethodDelete:
+ bucketDeleted = true
+ return s3Response(http.StatusNoContent, ""), nil
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ if err := storage.RemoveBucket(t.Context(), "kb01"); err == nil {
+ t.Fatal("RemoveBucket() returned nil after cleanup failure")
+ }
+ if bucketDeleted {
+ t.Fatal("RemoveBucket() deleted the physical bucket after cleanup failure")
+ }
+ })
+ }
+}
+
+func TestS3StorageRemoveBucketHeadBucketErrors(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ status int
+ code string
+ wantErr bool
+ }{
+ {"missing bucket", http.StatusNotFound, "NotFound", false},
+ {"access denied", http.StatusForbidden, "AccessDenied", true},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ destructiveRequest := false
+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ if req.Method == http.MethodHead {
+ return s3Response(test.status, `<Error><Code>`+test.code+`</Code></Error>`), nil
+ }
+ destructiveRequest = true
+ return s3Response(http.StatusInternalServerError, ""), nil
+ })
+ err := storage.RemoveBucket(t.Context(), "kb01")
+ if (err != nil) != test.wantErr {
+ t.Fatalf("RemoveBucket() error = %v, want error: %t", err, test.wantErr)
+ }
+ if destructiveRequest {
+ t.Fatal("RemoveBucket() performed cleanup after HeadBucket failure")
+ }
+ })
+ }
+}