RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(storage): add Content-MD5 to S3 batch delete requests (#21877)
liuzelei4bf54dbc2602
diff --git a/internal/storage/s3.go b/internal/storage/s3.goindex f95f503eee..823c85f760 100644--- a/internal/storage/s3.go+++ b/internal/storage/s3.go@@ -30,6 +30,7 @@ import ("github.com/aws/aws-sdk-go-v2/service/s3""github.com/aws/aws-sdk-go-v2/service/s3/types""github.com/aws/smithy-go"+ smithyhttp "github.com/aws/smithy-go/transport/http""go.uber.org/zap")@@ -240,12 +241,19 @@ func (s *S3Storage) removeObjects(ctx context.Context, bucket, prefix string) erreturn nil}+// deleteObjects deletes the supplied objects in batches of at most 1,000.+// It includes Content-MD5 alongside the SDK checksum and returns on the first+// request-level or per-object error.func (s *S3Storage) deleteObjects(ctx context.Context, bucket string, objects []types.ObjectIdentifier) error {for len(objects) > 0 {count := min(len(objects), 1000)output, err := s.client.DeleteObjects(ctx, &s3.DeleteObjectsInput{Bucket: aws.String(bucket),Delete: &types.Delete{Objects: objects[:count], Quiet: aws.Bool(true)},+ }, func(options *s3.Options) {+ // COS requires Content-MD5 for batch deletion in addition to the+ // newer SDK's CRC32 checksum. Keep both on this operation only.+ options.APIOptions = append(options.APIOptions, smithyhttp.AddContentChecksumMiddleware)})if err != nil {return fmt.Errorf("delete S3 objects in %q: %w", bucket, err)diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.goindex d05f71b855..1ccd6513bf 100644--- a/internal/storage/s3_test.go+++ b/internal/storage/s3_test.go@@ -3,8 +3,13 @@ package storageimport ("bytes""context"+ "crypto/md5""crypto/sha256"+ "encoding/base64"+ "encoding/binary"+ "encoding/xml""fmt"+ "hash/crc32""io""net/http""net/http/httptest"@@ -51,6 +56,72 @@ func s3Response(status int, body string) *http.Response {return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}}+// TestS3DeleteObjectsAddsContentMD5 checks that Content-MD5 covers the serialized+// XML and is signed without replacing CRC32 or suppressing per-object errors.+func TestS3DeleteObjectsAddsContentMD5(t *testing.T) {+ for _, tc := range []struct {+ name, response string+ wantError bool+ }{+ {name: "success", response: `<DeleteResult/>`},+ {name: "per-object error", response: `<DeleteResult><Error><Key>docs/a&b.txt</Key><VersionId>null</VersionId><Code>AccessDenied</Code><Message>not allowed</Message></Error></DeleteResult>`, wantError: true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ requests := 0+ store := newS3TestStorage("ap-shanghai", func(req *http.Request) (*http.Response, error) {+ requests+++ if req.Method != http.MethodPost || !req.URL.Query().Has("delete") {+ t.Fatal("expected a batch delete request")+ }+ body, err := io.ReadAll(req.Body)+ if err != nil {+ t.Fatal(err)+ }+ md5Sum := md5.Sum(body)+ if got, want := req.Header.Get("Content-MD5"), base64.StdEncoding.EncodeToString(md5Sum[:]); got != want {+ t.Errorf("Content-MD5 = %q, want digest of serialized XML %q", got, want)+ }+ crc := make([]byte, 4)+ binary.BigEndian.PutUint32(crc, crc32.ChecksumIEEE(body))+ if got, want := req.Header.Get("X-Amz-Checksum-Crc32"), base64.StdEncoding.EncodeToString(crc); got != want {+ t.Error("SDK CRC32 checksum was lost or changed")+ }+ if !strings.Contains(req.Header.Get("Authorization"), "content-md5") {+ t.Error("Content-MD5 was added after signing")+ }+ var deletion struct {+ Objects []struct {+ Key string+ VersionID string `xml:"VersionId"`+ } `xml:"Object"`+ Quiet bool+ }+ if err := xml.Unmarshal(body, &deletion); err != nil {+ t.Fatal(err)+ }+ if len(deletion.Objects) != 2 || deletion.Objects[0].Key != "docs/a&b.txt" || deletion.Objects[0].VersionID != "null" || deletion.Objects[1].Key != "docs/说明.txt" || deletion.Objects[1].VersionID != "version-2" || !deletion.Quiet {+ t.Error("batch object identifiers or quiet mode changed")+ }+ return s3Response(http.StatusOK, tc.response), nil+ })+ err := store.deleteObjects(t.Context(), "bucket", []types.ObjectIdentifier{+ {Key: aws.String("docs/a&b.txt"), VersionId: aws.String("null")},+ {Key: aws.String("docs/说明.txt"), VersionId: aws.String("version-2")},+ })+ if requests != 1 {+ t.Fatalf("requests = %d, want 1", requests)+ }+ if tc.wantError {+ if err == nil || !strings.Contains(err.Error(), "AccessDenied") || !strings.Contains(err.Error(), "not allowed") {+ t.Fatal("per-object deletion error was not propagated")+ }+ } else if err != nil {+ t.Fatal(err)+ }+ })+ }+}+func TestS3StorageConnectAppliesAddressingStyle(t *testing.T) {for _, test := range []struct {name string