Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(storage): use virtual-hosted addressing for Tencent COS through the S3 driver (#3722)

Ankit Jha27b9ee74a02f
diff --git a/internal/application/service/file/s3.go b/internal/application/service/file/s3.go
index e321a1f69..634297232 100644
--- a/internal/application/service/file/s3.go
+++ b/internal/application/service/file/s3.go
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"mime/multipart"
+ "net/url"
"path/filepath"
"strings"
"time"
@@ -29,6 +30,23 @@ type s3FileService struct {
pathPrefix string
}
+// s3UsePathStyle reports whether the S3 client should use path-style
+// addressing. S3-compatible services default to path-style; AWS and Tencent
+// COS use virtual-hosted style, since COS rejects path-style for buckets
+// created from 2024-01-01 (#3134). force_path_style always wins.
+func s3UsePathStyle(endpoint string, forcePathStyle bool) bool {
+ if forcePathStyle {
+ return true
+ }
+ if strings.Contains(endpoint, "amazonaws.com") {
+ return false
+ }
+ if u, err := url.Parse(endpoint); err == nil && strings.HasSuffix(strings.ToLower(u.Hostname()), ".myqcloud.com") {
+ return false
+ }
+ return true
+}
+
// newS3Client creates a bare s3FileService with just the SDK client initialised.
func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix string, forcePathStyle bool) (*s3FileService, error) {
if err := utils.ValidateURLForSSRF(endpoint); err != nil {
@@ -56,15 +74,12 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
}
// Create S3 client with custom endpoint if provided.
- // For S3-compatible services (non-AWS), use path-style addressing
- // (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
httpClient := objectStorageHTTPClient()
var client *s3.Client
if endpoint != "" {
- usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(endpoint)
- o.UsePathStyle = usePathStyle
+ o.UsePathStyle = s3UsePathStyle(endpoint, forcePathStyle)
if !strings.Contains(endpoint, "amazonaws.com") {
// S3-compatible services commonly reject the SDK's default
// trailing checksum negotiation. Only relax this for explicit
diff --git a/internal/application/service/file/s3_test.go b/internal/application/service/file/s3_test.go
index 0291e8b3c..9d57ff6b8 100644
--- a/internal/application/service/file/s3_test.go
+++ b/internal/application/service/file/s3_test.go
@@ -2,11 +2,13 @@ package file
import (
"context"
+ "net/http"
"strings"
"testing"
"github.com/Tencent/WeKnora/internal/utils"
"github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
)
func TestNewS3Client_Credentials(t *testing.T) {
@@ -48,11 +50,12 @@ func TestNewS3Client_Credentials(t *testing.T) {
})
}
-func TestNewS3Client_PathStyleForCompatibleEndpoints(t *testing.T) {
+func TestS3UsePathStyle(t *testing.T) {
tests := []struct {
- name string
- endpoint string
- wantPathStyle bool
+ name string
+ endpoint string
+ forcePathStyle bool
+ wantPathStyle bool
}{
{
name: "S3-compatible service uses path-style",
@@ -74,19 +77,65 @@ func TestNewS3Client_PathStyleForCompatibleEndpoints(t *testing.T) {
endpoint: "https://s3.cn-north-1.amazonaws.com.cn",
wantPathStyle: false,
},
+ {
+ name: "Tencent COS endpoint uses virtual-hosted",
+ endpoint: "https://cos.ap-shanghai.myqcloud.com",
+ wantPathStyle: false,
+ },
+ {
+ name: "Tencent COS endpoint honors force_path_style",
+ endpoint: "https://cos.ap-shanghai.myqcloud.com",
+ forcePathStyle: true,
+ wantPathStyle: true,
+ },
+ {
+ name: "lookalike host is not treated as COS",
+ endpoint: "https://cos.myqcloud.com.example.org",
+ wantPathStyle: true,
+ },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- // Verify the path-style detection logic matches our implementation
- usePathStyle := !strings.Contains(tt.endpoint, "amazonaws.com")
- if usePathStyle != tt.wantPathStyle {
- t.Errorf("endpoint %q: usePathStyle = %v, want %v", tt.endpoint, usePathStyle, tt.wantPathStyle)
+ if got := s3UsePathStyle(tt.endpoint, tt.forcePathStyle); got != tt.wantPathStyle {
+ t.Errorf("s3UsePathStyle(%q, %v) = %v, want %v", tt.endpoint, tt.forcePathStyle, got, tt.wantPathStyle)
}
})
}
}
+// Tencent COS rejects path-style requests for buckets created since
+// 2024-01-01, so the request must go to {bucket}.{host}/{key} (#3134).
+func TestNewS3Client_TencentCOSPutObjectUsesVirtualHostedURL(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "cos.ap-shanghai.myqcloud.com")
+ utils.ResetSSRFWhitelistForTest()
+ t.Cleanup(utils.ResetSSRFWhitelistForTest)
+
+ const bucket = "examplebucket-1250000000"
+ svc, err := newS3Client("https://cos.ap-shanghai.myqcloud.com", "ak", "sk", bucket, "ap-shanghai", "", false)
+ if err != nil {
+ t.Fatalf("newS3Client() error = %v", err)
+ }
+ trip := &capturingRoundTripper{}
+ opts := svc.client.Options()
+ opts.HTTPClient = &http.Client{Transport: trip}
+ opts.Retryer = aws.NopRetryer{}
+ _, err = s3.New(opts).PutObject(context.Background(), &s3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String("a/b.txt"),
+ Body: strings.NewReader("hello"),
+ })
+ if err != nil {
+ t.Fatalf("PutObject: %v", err)
+ }
+ if want := bucket + ".cos.ap-shanghai.myqcloud.com"; trip.host != want {
+ t.Errorf("host = %q, want %q", trip.host, want)
+ }
+ if want := "/a/b.txt"; trip.path != want {
+ t.Errorf("path = %q, want %q", trip.path, want)
+ }
+}
+
func TestNewS3Client_EmptyEndpoint(t *testing.T) {
// Empty endpoint should not trigger path-style (standard AWS S3)
endpoint := ""