Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(s3): honor signature_version and addressing_style for S3-compatible stores (#20815)

Paul Yao808841a91f37
diff --git a/internal/storage/s3.go b/internal/storage/s3.go
index cca3478841..a67c20cea1 100644
--- a/internal/storage/s3.go
+++ b/internal/storage/s3.go
@@ -26,6 +26,7 @@ import (
"time"
"github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
s3Config "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
@@ -83,16 +84,16 @@ func (s *S3Storage) connect(ctx context.Context) error {
return fmt.Errorf("failed to load AWS config: %w", err)
}
- // Create S3 client with custom endpoint if provided
- var clientOpts []func(*s3.Options)
- if s.config.EndpointURL != "" {
- clientOpts = append(clientOpts, func(o *s3.Options) {
+ s.client = s3.NewFromConfig(cfg, func(o *s3.Options) {
+ if s.config.EndpointURL != "" {
o.BaseEndpoint = aws.String(s.config.EndpointURL)
- o.UsePathStyle = s.config.AddressingStyle == "path"
- })
- }
-
- s.client = s3.NewFromConfig(cfg, clientOpts...)
+ }
+ o.UsePathStyle = s.config.AddressingStyle == "path"
+ if s.config.SignatureVersion == "s3" {
+ o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+ o.APIOptions = append(o.APIOptions, v4.SwapComputePayloadSHA256ForUnsignedPayloadMiddleware)
+ }
+ })
return nil
}
diff --git a/internal/storage/s3_test.go b/internal/storage/s3_test.go
index 621a0e5f85..75662525e3 100644
--- a/internal/storage/s3_test.go
+++ b/internal/storage/s3_test.go
@@ -1,8 +1,13 @@
package storage
import (
+ "bytes"
+ "crypto/sha256"
+ "fmt"
"io"
"net/http"
+ "net/http/httptest"
+ "ragflow/internal/server/config"
"strings"
"testing"
@@ -33,6 +38,137 @@ func s3Response(status int, body string) *http.Response {
return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body))}
}
+func TestS3StorageConnectAppliesAddressingStyle(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ addressingStyle string
+ wantPathStyle bool
+ }{
+ {name: "path", addressingStyle: "path", wantPathStyle: true},
+ {name: "virtual", addressingStyle: "virtual"},
+ {name: "unset"},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ storage, err := NewS3Storage(t.Context(), config.S3Config{
+ AccessKey: "access",
+ SecretKey: "secret",
+ Region: "us-east-1",
+ AddressingStyle: test.addressingStyle,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := storage.client.Options().UsePathStyle; got != test.wantPathStyle {
+ t.Fatalf("UsePathStyle = %t, want %t", got, test.wantPathStyle)
+ }
+ })
+ }
+}
+
+func TestS3StorageConnectConfiguresPayloadSigning(t *testing.T) {
+ t.Setenv("AWS_REQUEST_CHECKSUM_CALCULATION", "when_supported")
+ payload := []byte("content")
+ for _, test := range []struct {
+ name string
+ signatureVersion string
+ wantPayloadHash string
+ wantContentEncoding string
+ wantBody string
+ }{
+ {name: "s3", signatureVersion: "s3", wantPayloadHash: "UNSIGNED-PAYLOAD", wantBody: string(payload)},
+ {name: "v4", signatureVersion: "v4", wantPayloadHash: "STREAMING-UNSIGNED-PAYLOAD-TRAILER", wantContentEncoding: "aws-chunked"},
+ {name: "s3v4", signatureVersion: "s3v4", wantPayloadHash: "STREAMING-UNSIGNED-PAYLOAD-TRAILER", wantContentEncoding: "aws-chunked"},
+ {name: "unset", wantPayloadHash: "STREAMING-UNSIGNED-PAYLOAD-TRAILER", wantContentEncoding: "aws-chunked"},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ var gotPutPayloadHash string
+ var gotPutContentEncoding string
+ var gotPutBody string
+ var gotHeadPayloadHash string
+ var gotDeletePayloadHash string
+ var gotDeleteContentEncoding string
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ if req.URL.Query().Has("delete") {
+ gotDeletePayloadHash = req.Header.Get("x-amz-content-sha256")
+ gotDeleteContentEncoding = req.Header.Get("Content-Encoding")
+ _, _ = w.Write([]byte("<DeleteResult/>"))
+ return
+ }
+ switch req.Method {
+ case http.MethodPut:
+ gotPutPayloadHash = req.Header.Get("x-amz-content-sha256")
+ gotPutContentEncoding = req.Header.Get("Content-Encoding")
+ body, err := io.ReadAll(req.Body)
+ if err != nil {
+ t.Error(err)
+ }
+ gotPutBody = string(body)
+ case http.MethodHead:
+ gotHeadPayloadHash = req.Header.Get("x-amz-content-sha256")
+ }
+ w.WriteHeader(http.StatusOK)
+ }))
+ defer server.Close()
+
+ storage, err := NewS3Storage(t.Context(), config.S3Config{
+ AccessKey: "access",
+ SecretKey: "secret",
+ Region: "us-east-1",
+ EndpointURL: server.URL,
+ SignatureVersion: test.signatureVersion,
+ AddressingStyle: "path",
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ storage.client = s3.New(storage.client.Options(), func(options *s3.Options) {
+ options.HTTPClient = server.Client()
+ })
+ _, err = storage.client.PutObject(t.Context(), &s3.PutObjectInput{
+ Bucket: aws.String("bucket"),
+ Key: aws.String("object"),
+ Body: bytes.NewReader(payload),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = storage.client.HeadBucket(t.Context(), &s3.HeadBucketInput{Bucket: aws.String("bucket")}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = storage.client.DeleteObjects(t.Context(), &s3.DeleteObjectsInput{
+ Bucket: aws.String("bucket"),
+ Delete: &types.Delete{Objects: []types.ObjectIdentifier{{Key: aws.String("object")}}},
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if gotPutPayloadHash != test.wantPayloadHash {
+ t.Fatalf("PutObject x-amz-content-sha256 = %q, want %q", gotPutPayloadHash, test.wantPayloadHash)
+ }
+ if gotPutContentEncoding != test.wantContentEncoding {
+ t.Fatalf("PutObject Content-Encoding = %q, want %q", gotPutContentEncoding, test.wantContentEncoding)
+ }
+ if test.wantBody != "" && gotPutBody != test.wantBody {
+ t.Fatalf("PutObject body = %q, want %q", gotPutBody, test.wantBody)
+ }
+ wantHeadPayloadHash := fmt.Sprintf("%x", sha256.Sum256(nil))
+ if test.signatureVersion == "s3" {
+ wantHeadPayloadHash = "UNSIGNED-PAYLOAD"
+ }
+ if gotHeadPayloadHash != wantHeadPayloadHash {
+ t.Fatalf("HeadBucket x-amz-content-sha256 = %q, want %q", gotHeadPayloadHash, wantHeadPayloadHash)
+ }
+ if test.signatureVersion == "s3" {
+ if gotDeletePayloadHash != "UNSIGNED-PAYLOAD" {
+ t.Fatalf("DeleteObjects x-amz-content-sha256 = %q, want UNSIGNED-PAYLOAD", gotDeletePayloadHash)
+ }
+ if gotDeleteContentEncoding != "" {
+ t.Fatalf("DeleteObjects Content-Encoding = %q, want empty", gotDeleteContentEncoding)
+ }
+ }
+ })
+ }
+}
+
func TestS3StoragePutCreatesBucketInResolvedRegion(t *testing.T) {
for _, test := range []struct {
region string