RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(storage): handle versioned S3 bucket deletion (#19199)
Sbaaoui Idriss9f73625f532d
diff --git a/internal/storage/s3.go b/internal/storage/s3.goindex 78f4c376e3..60bd06a3cf 100644--- a/internal/storage/s3.go+++ b/internal/storage/s3.go@@ -29,6 +29,7 @@ import (s3Config "github.com/aws/aws-sdk-go-v2/config""github.com/aws/aws-sdk-go-v2/credentials""github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/aws-sdk-go-v2/service/s3/types""github.com/aws/smithy-go""go.uber.org/zap")@@ -334,53 +335,79 @@ func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {// RemoveBucket removes a bucket and all its objectsfunc (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {- actualBucket := bucket- if s.bucket != "" {- actualBucket = s.bucket+ exists, err := s.bucketExistsForRemoval(ctx, bucket)+ if err != nil {+ return err}-- // Check if bucket exists- if !s.BucketExists(ctx, actualBucket) {+ if !exists {return nil}- // List and delete all objects- listInput := &s3.ListObjectsV2Input{- Bucket: aws.String(actualBucket),+ if err := s.removeObjects(ctx, bucket); err != nil {+ return err}- for {- result, err := s.client.ListObjectsV2(ctx, listInput)- if err != nil {- common.Error("Failed to list objects", err, zap.String("bucket", actualBucket), zap.Error(err))- return err- }+ _, err = s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{+ Bucket: aws.String(bucket),+ })+ if err != nil {+ common.Error("Failed to delete bucket", err, zap.String("bucket", bucket), zap.Error(err))+ return err+ }- for _, obj := range result.Contents {- _, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{- Bucket: aws.String(actualBucket),- Key: obj.Key,- })- if err != nil {- common.Error("Failed to delete object", err, zap.String("bucket", actualBucket), zap.Error(err))- }- }+ return nil+}- if result.IsTruncated == nil || !*result.IsTruncated {- break- }- listInput.ContinuationToken = result.NextContinuationToken+func (s *S3Storage) bucketExistsForRemoval(ctx context.Context, bucket string) (bool, error) {+ _, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})+ if err == nil {+ return true, nil+ }+ if isS3BucketNotFound(err) {+ return false, nil}+ return false, fmt.Errorf("head S3 bucket %q: %w", bucket, err)+}- // Delete bucket- _, err := s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{- Bucket: aws.String(actualBucket),+func (s *S3Storage) removeObjects(ctx context.Context, bucket string) error {+ versions := s3.NewListObjectVersionsPaginator(s.client, &s3.ListObjectVersionsInput{+ Bucket: aws.String(bucket),})- if err != nil {- common.Error("Failed to delete bucket", err, zap.String("bucket", actualBucket), zap.Error(err))- return err+ for versions.HasMorePages() {+ page, err := versions.NextPage(ctx)+ if err != nil {+ return fmt.Errorf("list S3 object versions in %q: %w", bucket, err)+ }+ toDelete := make([]types.ObjectIdentifier, 0, len(page.Versions)+len(page.DeleteMarkers))+ for _, version := range page.Versions {+ toDelete = append(toDelete, types.ObjectIdentifier{Key: version.Key, VersionId: version.VersionId})+ }+ for _, marker := range page.DeleteMarkers {+ toDelete = append(toDelete, types.ObjectIdentifier{Key: marker.Key, VersionId: marker.VersionId})+ }+ if err := s.deleteObjects(ctx, bucket, toDelete); err != nil {+ return err+ }}+ return nil+}+func (s *S3Storage) deleteObjects(ctx context.Context, bucket string, objects []types.ObjectIdentifier) error {+ for len(objects) > 0 {+ count := min(len(objects), 1000)+ output, err := s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{+ Bucket: aws.String(bucket),+ Delete: &types.Delete{Objects: objects[:count], Quiet: aws.Bool(true)},+ })+ if err != nil {+ return fmt.Errorf("delete S3 objects in %q: %w", bucket, err)+ }+ if len(output.Errors) > 0 {+ failed := output.Errors[0]+ return fmt.Errorf("delete S3 object %q (version %q): %s: %s", aws.ToString(failed.Key), aws.ToString(failed.VersionId), aws.ToString(failed.Code), aws.ToString(failed.Message))+ }+ objects = objects[count:]+ }return nil}@@ -435,3 +462,14 @@ func isS3NotFound(err error) bool {}return false}++func isS3BucketNotFound(err error) bool {+ if err == nil {+ return false+ }+ var apiErr smithy.APIError+ if errors.As(err, &apiErr) {+ return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "404" || apiErr.ErrorCode() == "NoSuchBucket"+ }+ return false+}diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.gonew file mode 100644index 0000000000..113f947af9--- /dev/null+++ b/internal/storage/s3_test.go@@ -0,0 +1,234 @@+package storage++import (+ "io"+ "net/http"+ "strings"+ "testing"++ "github.com/aws/aws-sdk-go-v2/aws"+ "github.com/aws/aws-sdk-go-v2/credentials"+ "github.com/aws/aws-sdk-go-v2/service/s3"+ "github.com/aws/aws-sdk-go-v2/service/s3/types"+)++type s3RoundTripper func(*http.Request) (*http.Response, error)++func (f s3RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { return f(req) }++func newS3TestStorage(handler s3RoundTripper) *S3Storage {+ client := s3.NewFromConfig(aws.Config{+ Region: "us-east-1",+ Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider("access", "secret", "")),+ HTTPClient: &http.Client{Transport: handler},+ }, func(options *s3.Options) {+ options.BaseEndpoint = aws.String("https://s3.test")+ options.UsePathStyle = true+ options.RetryMaxAttempts = 1+ })+ return &S3Storage{client: client}+}++func s3Response(status int, body string) *http.Response {+ return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}+}++func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {+ bucketDeleted := false+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead:+ return s3Response(http.StatusOK, ""), nil+ case req.URL.Query().Has("versions"):+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated></ListVersionsResult>`), nil+ case req.Method == http.MethodDelete:+ if req.URL.Path != "/kb01" {+ t.Fatalf("DeleteBucket path = %q, want /kb01", req.URL.Path)+ }+ bucketDeleted = true+ return s3Response(http.StatusNoContent, ""), nil+ }+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ return nil, nil+ })+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {+ t.Fatal(err)+ }+ if !bucketDeleted {+ t.Fatal("RemoveBucket() did not delete the empty physical bucket")+ }+}++func TestS3StorageRemoveBucketDeletesVersionsAndMarkers(t *testing.T) {+ var deleteBody string+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead:+ return s3Response(http.StatusOK, ""), nil+ case req.URL.Query().Has("versions"):+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><Version><Key>current</Key><VersionId>v2</VersionId></Version><Version><Key>old</Key><VersionId>v1</VersionId></Version><DeleteMarker><Key>removed</Key><VersionId>m1</VersionId></DeleteMarker></ListVersionsResult>`), nil+ case req.URL.Query().Has("delete"):+ body, _ := io.ReadAll(req.Body)+ deleteBody = string(body)+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil+ case req.Method == http.MethodDelete:+ return s3Response(http.StatusNoContent, ""), nil+ }+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ return nil, nil+ })+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {+ t.Fatal(err)+ }+ for _, value := range []string{"<Key>current</Key><VersionId>v2</VersionId>", "<Key>old</Key><VersionId>v1</VersionId>", "<Key>removed</Key><VersionId>m1</VersionId>"} {+ if !strings.Contains(deleteBody, value) {+ t.Fatalf("DeleteObjects did not preserve %q: %s", value, deleteBody)+ }+ }+}++func TestS3StorageRemoveBucketDeletesNullVersion(t *testing.T) {+ var deleteBody string+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead:+ return s3Response(http.StatusOK, ""), nil+ case req.URL.Query().Has("versions"):+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><Version><Key>document</Key><VersionId>null</VersionId></Version></ListVersionsResult>`), nil+ case req.URL.Query().Has("delete"):+ body, _ := io.ReadAll(req.Body)+ deleteBody = string(body)+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil+ case req.Method == http.MethodDelete:+ return s3Response(http.StatusNoContent, ""), nil+ }+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ return nil, nil+ })+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(deleteBody, `<Key>document</Key><VersionId>null</VersionId>`) {+ t.Fatalf("DeleteObjects omitted the null version ID: %s", deleteBody)+ }+}++func TestS3StorageRemoveBucketVersionPagination(t *testing.T) {+ versionCalls := 0+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead:+ return s3Response(http.StatusOK, ""), nil+ case req.URL.Query().Has("versions"):+ versionCalls+++ if versionCalls == 1 {+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>true</IsTruncated><NextKeyMarker>a</NextKeyMarker><NextVersionIdMarker>v1</NextVersionIdMarker><Version><Key>a</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil+ }+ if req.URL.Query().Get("key-marker") != "a" || req.URL.Query().Get("version-id-marker") != "v1" {+ t.Fatalf("missing version continuation markers: %s", req.URL)+ }+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><DeleteMarker><Key>b</Key><VersionId>m1</VersionId></DeleteMarker></ListVersionsResult>`), nil+ case req.URL.Query().Has("delete"):+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil+ case req.Method == http.MethodDelete:+ return s3Response(http.StatusNoContent, ""), nil+ }+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ return nil, nil+ })+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {+ t.Fatal(err)+ }+ if versionCalls != 2 {+ t.Fatalf("ListObjectVersions calls = %d, want 2", versionCalls)+ }+}++func TestS3StorageRemoveBucketFailures(t *testing.T) {+ t.Run("batch limit", func(t *testing.T) {+ requests := 0+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ if !req.URL.Query().Has("delete") {+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ }+ requests+++ return s3Response(http.StatusOK, `<DeleteResult/>`), nil+ })+ objects := make([]types.ObjectIdentifier, 1001)+ for i := range objects {+ objects[i].Key = aws.String("key")+ }+ if err := storage.deleteObjects(t.Context(), "kb01", objects); err != nil {+ t.Fatal(err)+ }+ if requests != 2 {+ t.Fatalf("DeleteObjects requests = %d, want 2", requests)+ }+ })++ for _, test := range []struct {+ name, body string+ status int+ }{+ {"list failure", `<Error><Code>InternalError</Code></Error>`, http.StatusInternalServerError},+ {"embedded delete error", `<DeleteResult><Error><Key>a</Key><Code>AccessDenied</Code><Message>denied</Message></Error></DeleteResult>`, http.StatusOK},+ } {+ t.Run(test.name, func(t *testing.T) {+ bucketDeleted := false+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead:+ return s3Response(http.StatusOK, ""), nil+ case req.URL.Query().Has("versions"):+ if test.name == "list failure" {+ return s3Response(test.status, test.body), nil+ }+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><Version><Key>a</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil+ case req.URL.Query().Has("delete"):+ return s3Response(test.status, test.body), nil+ case req.Method == http.MethodDelete:+ bucketDeleted = true+ return s3Response(http.StatusNoContent, ""), nil+ }+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ return nil, nil+ })+ if err := storage.RemoveBucket(t.Context(), "kb01"); err == nil {+ t.Fatal("RemoveBucket() returned nil after cleanup failure")+ }+ if bucketDeleted {+ t.Fatal("RemoveBucket() deleted the physical bucket after cleanup failure")+ }+ })+ }+}++func TestS3StorageRemoveBucketHeadBucketErrors(t *testing.T) {+ for _, test := range []struct {+ name string+ status int+ code string+ wantErr bool+ }{+ {"missing bucket", http.StatusNotFound, "NotFound", false},+ {"access denied", http.StatusForbidden, "AccessDenied", true},+ } {+ t.Run(test.name, func(t *testing.T) {+ destructiveRequest := false+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ if req.Method == http.MethodHead {+ return s3Response(test.status, `<Error><Code>`+test.code+`</Code></Error>`), nil+ }+ destructiveRequest = true+ return s3Response(http.StatusInternalServerError, ""), nil+ })+ err := storage.RemoveBucket(t.Context(), "kb01")+ if (err != nil) != test.wantErr {+ t.Fatalf("RemoveBucket() error = %v, want error: %t", err, test.wantErr)+ }+ if destructiveRequest {+ t.Fatal("RemoveBucket() performed cleanup after HeadBucket failure")+ }+ })+ }+}