Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(storage): use virtual-hosted addressing for Tencent COS through the S3 driver (#3722)
Ankit Jha27b9ee74a02f
diff --git a/internal/application/service/file/s3.go b/internal/application/service/file/s3.goindex e321a1f69..634297232 100644--- a/internal/application/service/file/s3.go+++ b/internal/application/service/file/s3.go@@ -7,6 +7,7 @@ import ("fmt""io""mime/multipart"+ "net/url""path/filepath""strings""time"@@ -29,6 +30,23 @@ type s3FileService struct {pathPrefix string}+// s3UsePathStyle reports whether the S3 client should use path-style+// addressing. S3-compatible services default to path-style; AWS and Tencent+// COS use virtual-hosted style, since COS rejects path-style for buckets+// created from 2024-01-01 (#3134). force_path_style always wins.+func s3UsePathStyle(endpoint string, forcePathStyle bool) bool {+ if forcePathStyle {+ return true+ }+ if strings.Contains(endpoint, "amazonaws.com") {+ return false+ }+ if u, err := url.Parse(endpoint); err == nil && strings.HasSuffix(strings.ToLower(u.Hostname()), ".myqcloud.com") {+ return false+ }+ return true+}+// newS3Client creates a bare s3FileService with just the SDK client initialised.func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) {if err := utils.ValidateURLForSSRF(endpoint); err != nil {@@ -56,15 +74,12 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix}// Create S3 client with custom endpoint if provided.- // For S3-compatible services (non-AWS), use path-style addressing- // (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).httpClient := objectStorageHTTPClient()var client *s3.Clientif endpoint != "" {- usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")client = s3.NewFromConfig(cfg, func(o *s3.Options) {o.BaseEndpoint = aws.String(endpoint)- o.UsePathStyle = usePathStyle+ o.UsePathStyle = s3UsePathStyle(endpoint, forcePathStyle)if !strings.Contains(endpoint, "amazonaws.com") {// S3-compatible services commonly reject the SDK's default// trailing checksum negotiation. Only relax this for explicitdiff --git a/internal/application/service/file/s3_test.go b/internal/application/service/file/s3_test.goindex 0291e8b3c..9d57ff6b8 100644--- a/internal/application/service/file/s3_test.go+++ b/internal/application/service/file/s3_test.go@@ -2,11 +2,13 @@ package fileimport ("context"+ "net/http""strings""testing""github.com/Tencent/WeKnora/internal/utils""github.com/aws/aws-sdk-go-v2/aws"+ "github.com/aws/aws-sdk-go-v2/service/s3")func TestNewS3Client_Credentials(t *testing.T) {@@ -48,11 +50,12 @@ func TestNewS3Client_Credentials(t *testing.T) {})}-func TestNewS3Client_PathStyleForCompatibleEndpoints(t *testing.T) {+func TestS3UsePathStyle(t *testing.T) {tests := []struct {- name string- endpoint string- wantPathStyle bool+ name string+ endpoint string+ forcePathStyle bool+ wantPathStyle bool}{{name: "S3-compatible service uses path-style",@@ -74,19 +77,65 @@ func TestNewS3Client_PathStyleForCompatibleEndpoints(t *testing.T) {endpoint: "https://s3.cn-north-1.amazonaws.com.cn",wantPathStyle: false,},+ {+ name: "Tencent COS endpoint uses virtual-hosted",+ endpoint: "https://cos.ap-shanghai.myqcloud.com",+ wantPathStyle: false,+ },+ {+ name: "Tencent COS endpoint honors force_path_style",+ endpoint: "https://cos.ap-shanghai.myqcloud.com",+ forcePathStyle: true,+ wantPathStyle: true,+ },+ {+ name: "lookalike host is not treated as COS",+ endpoint: "https://cos.myqcloud.com.example.org",+ wantPathStyle: true,+ },}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- // Verify the path-style detection logic matches our implementation- usePathStyle := !strings.Contains(tt.endpoint, "amazonaws.com")- if usePathStyle != tt.wantPathStyle {- t.Errorf("endpoint %q: usePathStyle = %v, want %v", tt.endpoint, usePathStyle, tt.wantPathStyle)+ if got := s3UsePathStyle(tt.endpoint, tt.forcePathStyle); got != tt.wantPathStyle {+ t.Errorf("s3UsePathStyle(%q, %v) = %v, want %v", tt.endpoint, tt.forcePathStyle, got, tt.wantPathStyle)}})}}+// Tencent COS rejects path-style requests for buckets created since+// 2024-01-01, so the request must go to {bucket}.{host}/{key} (#3134).+func TestNewS3Client_TencentCOSPutObjectUsesVirtualHostedURL(t *testing.T) {+ t.Setenv("SSRF_WHITELIST", "cos.ap-shanghai.myqcloud.com")+ utils.ResetSSRFWhitelistForTest()+ t.Cleanup(utils.ResetSSRFWhitelistForTest)++ const bucket = "examplebucket-1250000000"+ svc, err := newS3Client("https://cos.ap-shanghai.myqcloud.com", "ak", "sk", bucket, "ap-shanghai", "", false)+ if err != nil {+ t.Fatalf("newS3Client() error = %v", err)+ }+ trip := &capturingRoundTripper{}+ opts := svc.client.Options()+ opts.HTTPClient = &http.Client{Transport: trip}+ opts.Retryer = aws.NopRetryer{}+ _, err = s3.New(opts).PutObject(context.Background(), &s3.PutObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String("a/b.txt"),+ Body: strings.NewReader("hello"),+ })+ if err != nil {+ t.Fatalf("PutObject: %v", err)+ }+ if want := bucket + ".cos.ap-shanghai.myqcloud.com"; trip.host != want {+ t.Errorf("host = %q, want %q", trip.host, want)+ }+ if want := "/a/b.txt"; trip.path != want {+ t.Errorf("path = %q, want %q", trip.path, want)+ }+}+func TestNewS3Client_EmptyEndpoint(t *testing.T) {// Empty endpoint should not trigger path-style (standard AWS S3)endpoint := ""