Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(storage): use virtual-hosted addressing for OBS bucket requests (#3273)

Fei.Chenbca3a9f448e9
diff --git a/internal/application/service/file/obs.go b/internal/application/service/file/obs.go
index 451d92db5..bc5df6e81 100644
--- a/internal/application/service/file/obs.go
+++ b/internal/application/service/file/obs.go
@@ -5,6 +5,8 @@ import (
"fmt"
"io"
"mime/multipart"
+ "net"
+ "net/url"
"os"
"path/filepath"
"strings"
@@ -28,16 +30,32 @@ type obsFileService struct {
proxyDomain string
}
-type obsEndpointResolver struct {
- url string
+// obsUsePathStyle reports whether the OBS client should keep path-style
+// addressing for this endpoint. Huawei Cloud OBS rejects path-style bucket
+// addressing on domain endpoints since 2023-12-30, so domain endpoints use
+// virtual-hosted style ({bucket}.{host}); IP-literal or unparseable endpoints
+// have no DNS label to prefix and keep path-style. See issue #3269.
+func obsUsePathStyle(endpoint string) bool {
+ u, err := url.Parse(endpoint)
+ if err != nil || u.Host == "" {
+ return true
+ }
+ return net.ParseIP(u.Hostname()) != nil
}
-func (r *obsEndpointResolver) ResolveEndpoint(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {
- return aws.Endpoint{
- URL: r.url,
- SigningRegion: region,
- HostnameImmutable: true,
- }, nil
+// obsS3Options builds the S3 client options shared by the file service and the
+// connectivity check.
+func obsS3Options(endpoint, region, accessKey, secretAccessKey string) s3.Options {
+ return s3.Options{
+ Region: region,
+ BaseEndpoint: aws.String(endpoint),
+ Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretAccessKey, ""),
+ // OBS is S3-compatible but commonly rejects the SDK's default
+ // trailing checksum negotiation; align with the S3 driver (s3.go).
+ RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
+ UsePathStyle: obsUsePathStyle(endpoint),
+ HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
+ }
}
func NewObsFileService(
@@ -48,13 +66,7 @@ func NewObsFileService(
return nil, fmt.Errorf("unsafe OBS endpoint: %w", err)
}
- client := s3.New(s3.Options{
- Region: region,
- EndpointResolver: &obsEndpointResolver{url: endpoint},
- Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""),
- UsePathStyle: true,
- HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
- })
+ client := s3.New(obsS3Options(endpoint, region, accessKeyID, secretAccessKey))
_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{
Bucket: aws.String(bucketName),
@@ -87,13 +99,7 @@ func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secr
checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
- client := s3.New(s3.Options{
- Region: region,
- EndpointResolver: &obsEndpointResolver{url: endpoint},
- Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
- UsePathStyle: true,
- HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
- })
+ client := s3.New(obsS3Options(endpoint, region, accessKey, secretKey))
_, err := client.HeadBucket(checkCtx, &s3.HeadBucketInput{
Bucket: aws.String(bucketName),
@@ -236,7 +242,19 @@ func (s *obsFileService) GetFileURL(ctx context.Context, filePath string) (strin
return s.proxyDomain + "/" + strings.TrimPrefix(objectKey, "/"), nil
}
- return fmt.Sprintf("%s/%s/%s", s.endpoint, s.bucketName, strings.TrimPrefix(objectKey, "/")), nil
+ // Mirror the client addressing so generated URLs stay reachable: OBS
+ // rejects path-style on domain endpoints (issue #3269), so serve
+ // {bucket}.{endpoint-host}/{key}; IP-literal endpoints keep path-style.
+ if obsUsePathStyle(s.endpoint) {
+ return fmt.Sprintf("%s/%s/%s", s.endpoint, s.bucketName, strings.TrimPrefix(objectKey, "/")), nil
+ }
+ endpointURL, err := url.Parse(s.endpoint)
+ if err != nil {
+ return "", fmt.Errorf("invalid OBS endpoint %q: %w", s.endpoint, err)
+ }
+ endpointURL.Host = s.bucketName + "." + endpointURL.Host
+ endpointURL.Path = "/" + strings.TrimPrefix(objectKey, "/")
+ return endpointURL.String(), nil
}
// CopyFile copies an existing OBS object to a new knowledge-owned object using a
diff --git a/internal/application/service/file/obs_test.go b/internal/application/service/file/obs_test.go
new file mode 100644
index 000000000..313bbba9c
--- /dev/null
+++ b/internal/application/service/file/obs_test.go
@@ -0,0 +1,198 @@
+package file
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "strings"
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+// capturingRoundTripper records the first HTTP request URL the S3 client
+// would send, without dialing the network. That is the invariant #3269
+// actually cares about: PutObject must hit {bucket}.{host}/{key}, not
+// {host}/{bucket}/{key}. Checking UsePathStyle alone is not enough — PR
+// #3272 set that flag while HostnameImmutable still forced path-style.
+type capturingRoundTripper struct {
+ host string
+ path string
+}
+
+func (c *capturingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+ if c.host == "" {
+ c.host = req.URL.Host
+ c.path = req.URL.Path
+ }
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: http.Header{"Etag": []string{`"test"`}},
+ Body: io.NopCloser(strings.NewReader("")),
+ Request: req,
+ Proto: "HTTP/1.1",
+ ProtoMajor: 1,
+ ProtoMinor: 1,
+ }, nil
+}
+
+func putObjectWithCapture(t *testing.T, endpoint, region, bucket, key string) (host, path string) {
+ t.Helper()
+ trip := &capturingRoundTripper{}
+ opts := obsS3Options(endpoint, region, "ak", "sk")
+ opts.HTTPClient = &http.Client{Transport: trip}
+ opts.Retryer = aws.NopRetryer{}
+ client := s3.New(opts)
+ _, err := client.PutObject(context.Background(), &s3.PutObjectInput{
+ Bucket: aws.String(bucket),
+ Key: aws.String(key),
+ Body: strings.NewReader("hello"),
+ })
+ if err != nil {
+ t.Fatalf("PutObject: %v", err)
+ }
+ if trip.host == "" {
+ t.Fatal("PutObject did not issue an HTTP request")
+ }
+ return trip.host, trip.path
+}
+
+func TestObsS3OptionsUseVirtualHostedStyleForDomainEndpoints(t *testing.T) {
+ const endpoint = "https://obs.cn-south-1.myhuaweicloud.com"
+ opts := obsS3Options(endpoint, "cn-south-1", "ak", "sk")
+
+ if opts.UsePathStyle {
+ t.Fatal("domain endpoint must use virtual-hosted addressing (issue #3269)")
+ }
+ if opts.BaseEndpoint == nil || *opts.BaseEndpoint != endpoint {
+ t.Fatalf("BaseEndpoint = %v, want %q", opts.BaseEndpoint, endpoint)
+ }
+ if opts.RequestChecksumCalculation != aws.RequestChecksumCalculationWhenRequired {
+ t.Fatalf("RequestChecksumCalculation = %v, want WhenRequired", opts.RequestChecksumCalculation)
+ }
+ //nolint:staticcheck // v1 resolver must stay unset; HostnameImmutable pinned path-style
+ if opts.EndpointResolver != nil {
+ t.Fatal("deprecated custom resolver must stay dropped")
+ }
+ if opts.HTTPClient == nil {
+ t.Fatal("SSRF-safe HTTP client must be configured")
+ }
+}
+
+func TestObsS3OptionsKeepPathStyleForIPLiteralEndpoints(t *testing.T) {
+ for _, endpoint := range []string{
+ "http://192.168.1.10:9000",
+ "https://[2001:db8::1]:443",
+ "not a url",
+ } {
+ if opts := obsS3Options(endpoint, "region", "ak", "sk"); !opts.UsePathStyle {
+ t.Fatalf("endpoint %q: expected defensive path-style fallback", endpoint)
+ }
+ }
+}
+
+func TestObsPutObjectUsesVirtualHostedURL(t *testing.T) {
+ host, path := putObjectWithCapture(t,
+ "https://obs.cn-south-1.myhuaweicloud.com",
+ "cn-south-1",
+ "my-bucket",
+ "42/kb/file.pdf",
+ )
+ wantHost := "my-bucket.obs.cn-south-1.myhuaweicloud.com"
+ if host != wantHost {
+ t.Fatalf("PutObject host = %q, want virtual-hosted %q", host, wantHost)
+ }
+ if path != "/42/kb/file.pdf" {
+ t.Fatalf("PutObject path = %q, want /42/kb/file.pdf", path)
+ }
+}
+
+func TestObsPutObjectKeepsPathStyleForIPLiteralEndpoints(t *testing.T) {
+ host, path := putObjectWithCapture(t,
+ "http://192.168.1.10:9000",
+ "region",
+ "my-bucket",
+ "key.bin",
+ )
+ if host != "192.168.1.10:9000" {
+ t.Fatalf("PutObject host = %q, want 192.168.1.10:9000", host)
+ }
+ if path != "/my-bucket/key.bin" {
+ t.Fatalf("PutObject path = %q, want /my-bucket/key.bin", path)
+ }
+}
+
+func TestObsGetFileURLVirtualHostedStyle(t *testing.T) {
+ s := &obsFileService{
+ bucketName: "my-bucket",
+ endpoint: "https://obs.cn-south-1.myhuaweicloud.com",
+ }
+
+ got, err := s.GetFileURL(context.Background(), "obs://my-bucket/42/kb/file.pdf")
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := "https://my-bucket.obs.cn-south-1.myhuaweicloud.com/42/kb/file.pdf"
+ if got != want {
+ t.Fatalf("GetFileURL = %q, want %q", got, want)
+ }
+}
+
+func TestObsGetFileURLPreservesEndpointPort(t *testing.T) {
+ s := &obsFileService{
+ bucketName: "my-bucket",
+ endpoint: "https://obs.example.com:8443",
+ }
+
+ got, err := s.GetFileURL(context.Background(), "obs://my-bucket/key.bin")
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := "https://my-bucket.obs.example.com:8443/key.bin"
+ if got != want {
+ t.Fatalf("GetFileURL = %q, want %q", got, want)
+ }
+}
+
+func TestObsGetFileURLKeepsPathStyleForIPEndpoints(t *testing.T) {
+ s := &obsFileService{
+ bucketName: "my-bucket",
+ endpoint: "http://192.168.1.10:9000",
+ }
+
+ got, err := s.GetFileURL(context.Background(), "obs://my-bucket/key.bin")
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := "http://192.168.1.10:9000/my-bucket/key.bin"
+ if got != want {
+ t.Fatalf("GetFileURL = %q, want %q", got, want)
+ }
+}
+
+func TestObsGetFileURLProxyDomainAndHTTPPassthrough(t *testing.T) {
+ proxy := &obsFileService{
+ bucketName: "my-bucket",
+ proxyDomain: "https://cdn.example.com",
+ }
+ got, err := proxy.GetFileURL(context.Background(), "https://cdn.example.com/42/kb/file.pdf")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != "https://cdn.example.com/42/kb/file.pdf" {
+ t.Fatalf("proxy GetFileURL = %q", got)
+ }
+
+ s := &obsFileService{
+ bucketName: "my-bucket",
+ endpoint: "https://obs.cn-south-1.myhuaweicloud.com",
+ }
+ got, err = s.GetFileURL(context.Background(), "https://elsewhere.example.com/file")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != "https://elsewhere.example.com/file" {
+ t.Fatalf("http(s) passthrough = %q", got)
+ }
+}