Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(storage): add Content-MD5 to S3 batch delete requests (#21877)

liuzelei4bf54dbc2602
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index f95f503eee..823c85f760 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -30,6 +30,7 @@ import (
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
+ smithyhttp "github.com/aws/smithy-go/transport/http"
"go.uber.org/zap"
)
@@ -240,12 +241,19 @@ func (s *S3Storage) removeObjects(ctx context.Context, bucket, prefix string) er
return nil
}
+// deleteObjects deletes the supplied objects in batches of at most 1,000.
+// It includes Content-MD5 alongside the SDK checksum and returns on the first
+// request-level or per-object error.
func (s *S3Storage) deleteObjects(ctx context.Context, bucket string, objects []types.ObjectIdentifier) error {
for len(objects) > 0 {
count := min(len(objects), 1000)
output, err := s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
Bucket: aws.String(bucket),
Delete: &types.Delete{Objects: objects[:count], Quiet: aws.Bool(true)},
+ }, func(options *s3.Options) {
+ // COS requires Content-MD5 for batch deletion in addition to the
+ // newer SDK's CRC32 checksum. Keep both on this operation only.
+ options.APIOptions = append(options.APIOptions, smithyhttp.AddContentChecksumMiddleware)
})
if err != nil {
return fmt.Errorf("delete S3 objects in %q: %w", bucket, err)
diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go
index d05f71b855..1ccd6513bf 100644
--- a/internal/storage/s3_test.go
+++ b/internal/storage/s3_test.go
@@ -3,8 +3,13 @@ package storage
import (
"bytes"
"context"
+ "crypto/md5"
"crypto/sha256"
+ "encoding/base64"
+ "encoding/binary"
+ "encoding/xml"
"fmt"
+ "hash/crc32"
"io"
"net/http"
"net/http/httptest"
@@ -51,6 +56,72 @@ func s3Response(status int, body string) *http.Response {
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}
}
+// TestS3DeleteObjectsAddsContentMD5 checks that Content-MD5 covers the serialized
+// XML and is signed without replacing CRC32 or suppressing per-object errors.
+func TestS3DeleteObjectsAddsContentMD5(t *testing.T) {
+ for _, tc := range []struct {
+ name, response string
+ wantError bool
+ }{
+ {name: "success", response: `<DeleteResult/>`},
+ {name: "per-object error", response: `<DeleteResult><Error><Key>docs/a&amp;b.txt</Key><VersionId>null</VersionId><Code>AccessDenied</Code><Message>not allowed</Message></Error></DeleteResult>`, wantError: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ requests := 0
+ store := newS3TestStorage("ap-shanghai", func(req *http.Request) (*http.Response, error) {
+ requests++
+ if req.Method != http.MethodPost || !req.URL.Query().Has("delete") {
+ t.Fatal("expected a batch delete request")
+ }
+ body, err := io.ReadAll(req.Body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ md5Sum := md5.Sum(body)
+ if got, want := req.Header.Get("Content-MD5"), base64.StdEncoding.EncodeToString(md5Sum[:]); got != want {
+ t.Errorf("Content-MD5 = %q, want digest of serialized XML %q", got, want)
+ }
+ crc := make([]byte, 4)
+ binary.BigEndian.PutUint32(crc, crc32.ChecksumIEEE(body))
+ if got, want := req.Header.Get("X-Amz-Checksum-Crc32"), base64.StdEncoding.EncodeToString(crc); got != want {
+ t.Error("SDK CRC32 checksum was lost or changed")
+ }
+ if !strings.Contains(req.Header.Get("Authorization"), "content-md5") {
+ t.Error("Content-MD5 was added after signing")
+ }
+ var deletion struct {
+ Objects []struct {
+ Key string
+ VersionID string `xml:"VersionId"`
+ } `xml:"Object"`
+ Quiet bool
+ }
+ if err := xml.Unmarshal(body, &deletion); err != nil {
+ t.Fatal(err)
+ }
+ if len(deletion.Objects) != 2 || deletion.Objects[0].Key != "docs/a&b.txt" || deletion.Objects[0].VersionID != "null" || deletion.Objects[1].Key != "docs/说明.txt" || deletion.Objects[1].VersionID != "version-2" || !deletion.Quiet {
+ t.Error("batch object identifiers or quiet mode changed")
+ }
+ return s3Response(http.StatusOK, tc.response), nil
+ })
+ err := store.deleteObjects(t.Context(), "bucket", []types.ObjectIdentifier{
+ {Key: aws.String("docs/a&b.txt"), VersionId: aws.String("null")},
+ {Key: aws.String("docs/说明.txt"), VersionId: aws.String("version-2")},
+ })
+ if requests != 1 {
+ t.Fatalf("requests = %d, want 1", requests)
+ }
+ if tc.wantError {
+ if err == nil || !strings.Contains(err.Error(), "AccessDenied") || !strings.Contains(err.Error(), "not allowed") {
+ t.Fatal("per-object deletion error was not propagated")
+ }
+ } else if err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
func TestS3StorageConnectAppliesAddressingStyle(t *testing.T) {
for _, test := range []struct {
name string