Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(storage): scope configured single-bucket S3 removals correctly (#19217)

Sbaaoui Idrissec0cf792a948
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index 60bd06a3cf..96798c0f91 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -45,7 +45,9 @@ type S3Storage struct {
// NewS3Storage creates a new S3 storage instance
func NewS3Storage(ctx context.Context, config config.S3Config) (*S3Storage, error) {
storage := &S3Storage{
- config: config,
+ bucket: config.Bucket,
+ prefixPath: config.PrefixPath,
+ config: config,
}
if err := storage.connect(ctx); err != nil {
@@ -105,7 +107,13 @@ func (s *S3Storage) resolveBucketAndPath(bucket, fnm string) (string, string) {
}
actualPath := fnm
- if s.prefixPath != "" {
+ if s.bucket != "" {
+ prefix := s.prefixPath
+ if prefix != "" {
+ prefix += "/"
+ }
+ actualPath = fmt.Sprintf("%s%s/%s", prefix, bucket, fnm)
+ } else if s.prefixPath != "" {
actualPath = fmt.Sprintf("%s/%s/%s", s.prefixPath, bucket, fnm)
}
@@ -335,7 +343,8 @@ func (s *S3Storage) BucketExists(ctx context.Context, bucket string) bool {
// RemoveBucket removes a bucket and all its objects
func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
- exists, err := s.bucketExistsForRemoval(ctx, bucket)
+ actualBucket, prefix := s.resolveBucketAndPrefix(bucket)
+ exists, err := s.bucketExistsForRemoval(ctx, actualBucket)
if err != nil {
return err
}
@@ -343,12 +352,15 @@ func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
- if err := s.removeObjects(ctx, bucket); err != nil {
+ if err := s.removeObjects(ctx, actualBucket, prefix); err != nil {
return err
}
+ if s.bucket != "" {
+ return nil
+ }
_, err = s.client.DeleteBucket(ctx, &s3.DeleteBucketInput{
- Bucket: aws.String(bucket),
+ Bucket: aws.String(actualBucket),
})
if err != nil {
common.Error("Failed to delete bucket", err, zap.String("bucket", bucket), zap.Error(err))
@@ -358,6 +370,17 @@ func (s *S3Storage) RemoveBucket(ctx context.Context, bucket string) error {
return nil
}
+func (s *S3Storage) resolveBucketAndPrefix(bucket string) (string, string) {
+ if s.bucket == "" {
+ return bucket, ""
+ }
+ prefix := s.prefixPath
+ if prefix != "" {
+ prefix += "/"
+ }
+ return s.bucket, prefix + bucket + "/"
+}
+
func (s *S3Storage) bucketExistsForRemoval(ctx context.Context, bucket string) (bool, error) {
_, err := s.client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: aws.String(bucket)})
if err == nil {
@@ -369,9 +392,10 @@ func (s *S3Storage) bucketExistsForRemoval(ctx context.Context, bucket string) (
return false, fmt.Errorf("head S3 bucket %q: %w", bucket, err)
}
-func (s *S3Storage) removeObjects(ctx context.Context, bucket string) error {
+func (s *S3Storage) removeObjects(ctx context.Context, bucket, prefix string) error {
versions := s3.NewListObjectVersionsPaginator(s.client, &s3.ListObjectVersionsInput{
Bucket: aws.String(bucket),
+ Prefix: aws.String(prefix),
})
for versions.HasMorePages() {
page, err := versions.NextPage(ctx)
diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go
index 113f947af9..a4cdae9caa 100644
--- a/internal/storage/s3_test.go
+++ b/internal/storage/s3_test.go
@@ -59,6 +59,40 @@ func TestS3StorageRemoveBucketDeletesEmptyPhysicalBucket(t *testing.T) {
}
}
+func TestS3StorageRemoveBucketSingleBucketMode(t *testing.T) {
+ var deleteBody string
+ storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {
+ switch {
+ case req.Method == http.MethodHead:
+ if req.URL.Path != "/physical" {
+ t.Fatalf("HeadBucket path = %q, want /physical", req.URL.Path)
+ }
+ return s3Response(http.StatusOK, ""), nil
+ case req.URL.Query().Has("versions"):
+ if req.URL.Query().Get("prefix") != "prefix/kb01/" {
+ t.Fatalf("ListObjectVersions prefix = %q, want prefix/kb01/", req.URL.Query().Get("prefix"))
+ }
+ return s3Response(http.StatusOK, `<ListVersionsResult><IsTruncated>false</IsTruncated><Version><Key>prefix/kb01/document</Key><VersionId>v1</VersionId></Version></ListVersionsResult>`), nil
+ case req.URL.Query().Has("delete"):
+ body, _ := io.ReadAll(req.Body)
+ deleteBody = string(body)
+ return s3Response(http.StatusOK, `<DeleteResult/>`), nil
+ case req.Method == http.MethodDelete:
+ t.Fatalf("RemoveBucket deleted physical bucket in single-bucket mode")
+ }
+ t.Fatalf("unexpected request: %s %s", req.Method, req.URL)
+ return nil, nil
+ })
+ storage.bucket = "physical"
+ storage.prefixPath = "prefix"
+ if err := storage.RemoveBucket(t.Context(), "kb01"); err != nil {
+ t.Fatal(err)
+ }
+ if !strings.Contains(deleteBody, `<Key>prefix/kb01/document</Key><VersionId>v1</VersionId>`) {
+ t.Fatalf("DeleteObjects omitted matching version: %s", deleteBody)
+ }
+}
+
func TestS3StorageRemoveBucketDeletesVersionsAndMarkers(t *testing.T) {
var deleteBody string
storage := newS3TestStorage(func(req *http.Request) (*http.Response, error) {