RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(storage): scope configured single-bucket S3 removals correctly (#19217)
Sbaaoui Idrissec0cf792a948
diff --git a/internal/storage/s3.go b/internal/storage/s3.goindex 60bd06a3cf..96798c0f91 100644--- a/internal/storage/s3.go+++ b/internal/storage/s3.go@@ -45,7 +45,9 @@ type S3Storage struct {// NewS3Storage creates a new S3 storage instancefunc NewS3Storage(ctx context.Context, config config.S3Config) (*S3Storage, error) {storage := &S3Storage{- config: config,+ bucket: config.Bucket,+ prefixPath: config.PrefixPath,+ config: config,}if err := storage.connect(ctx); err != nil {@@ -105,7 +107,13 @@ func (s *S3Storage) resolveBucketAndPath(bucket, fnm string) (string, string) {}actualPath := fnm- if s.prefixPath != "" {+ if s.bucket != "" {+ prefix := s.prefixPath+ if prefix != "" {+ prefix += "/"+ }+ actualPath = fmt.Sprintf("%s%s/%s", prefix, bucket, fnm)+ } else if s.prefixPath != "" {actualPath = fmt.Sprintf("%s/%s/%s", s.prefixPath, bucket, fnm)}@@ -335,7 +343,8 @@ func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {// RemoveBucket removes a bucket and all its objectsfunc (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {- exists, err := s.bucketExistsForRemoval(ctx, bucket)+ actualBucket, prefix := s.resolveBucketAndPrefix(bucket)+ exists, err := s.bucketExistsForRemoval(ctx, actualBucket)if err != nil {return err}@@ -343,12 +352,15 @@ func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {return nil}- if err := s.removeObjects(ctx, bucket); err != nil {+ if err := s.removeObjects(ctx, actualBucket, prefix); err != nil {return err}+ if s.bucket != "" {+ return nil+ }_, err = s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{- Bucket: aws.String(bucket),+ Bucket: aws.String(actualBucket),})if err != nil {common.Error("Failed to delete bucket", err, zap.String("bucket", bucket), zap.Error(err))@@ -358,6 +370,17 @@ func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {return nil}+func (s *S3Storage) resolveBucketAndPrefix(bucket string) (string, string) {+ if s.bucket == "" {+ return bucket, ""+ }+ prefix := s.prefixPath+ if prefix != "" {+ prefix += "/"+ }+ return s.bucket, prefix + bucket + "/"+}+func (s *S3Storage) bucketExistsForRemoval(ctx context.Context, bucket string) (bool, error) {_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})if err == nil {@@ -369,9 +392,10 @@ func (s *S3Storage) bucketExistsForRemoval(ctx context.Context, bucket string) (return false, fmt.Errorf("head S3 bucket %q: %w", bucket, err)}-func (s *S3Storage) removeObjects(ctx context.Context, bucket string) error {+func (s *S3Storage) removeObjects(ctx context.Context, bucket, prefix string) error {versions := s3.NewListObjectVersionsPaginator(s.client, &s3.ListObjectVersionsInput{Bucket: aws.String(bucket),+ Prefix: aws.String(prefix),})for versions.HasMorePages() {page, err := versions.NextPage(ctx)diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.goindex 113f947af9..a4cdae9caa 100644--- a/internal/storage/s3_test.go+++ b/internal/storage/s3_test.go@@ -59,6 +59,40 @@ func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {}}+func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {+ var deleteBody string+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {+ switch {+ case req.Method == http.MethodHead:+ if req.URL.Path != "/physical" {+ t.Fatalf("HeadBucket path = %q, want /physical", req.URL.Path)+ }+ return s3Response(http.StatusOK, ""), nil+ case req.URL.Query().Has("versions"):+ if req.URL.Query().Get("prefix") != "prefix/kb01/" {+ t.Fatalf("ListObjectVersions prefix = %q, want prefix/kb01/", req.URL.Query().Get("prefix"))+ }+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><Version><Key>prefix/kb01/document</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil+ case req.URL.Query().Has("delete"):+ body, _ := io.ReadAll(req.Body)+ deleteBody = string(body)+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil+ case req.Method == http.MethodDelete:+ t.Fatalf("RemoveBucket deleted physical bucket in single-bucket mode")+ }+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)+ return nil, nil+ })+ storage.bucket = "physical"+ storage.prefixPath = "prefix"+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(deleteBody, `<Key>prefix/kb01/document</Key><VersionId>v1</VersionId>`) {+ t.Fatalf("DeleteObjects omitted matching version: %s", deleteBody)+ }+}+func TestS3StorageRemoveBucketDeletesVersionsAndMarkers(t *testing.T) {var deleteBody stringstorage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {