Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(storage): use virtual-hosted addressing for OBS bucket requests (#3273)
Fei.Chenbca3a9f448e9
diff --git a/internal/application/service/file/obs.go b/internal/application/service/file/obs.goindex 451d92db5..bc5df6e81 100644--- a/internal/application/service/file/obs.go+++ b/internal/application/service/file/obs.go@@ -5,6 +5,8 @@ import ("fmt""io""mime/multipart"+ "net"+ "net/url""os""path/filepath""strings"@@ -28,16 +30,32 @@ type obsFileService struct {proxyDomain string}-type obsEndpointResolver struct {- url string+// obsUsePathStyle reports whether the OBS client should keep path-style+// addressing for this endpoint. Huawei Cloud OBS rejects path-style bucket+// addressing on domain endpoints since 2023-12-30, so domain endpoints use+// virtual-hosted style ({bucket}.{host}); IP-literal or unparseable endpoints+// have no DNS label to prefix and keep path-style. See issue #3269.+func obsUsePathStyle(endpoint string) bool {+ u, err := url.Parse(endpoint)+ if err != nil || u.Host == "" {+ return true+ }+ return net.ParseIP(u.Hostname()) != nil}-func (r *obsEndpointResolver) ResolveEndpoint(region string, options s3.EndpointResolverOptions) (aws.Endpoint, error) {- return aws.Endpoint{- URL: r.url,- SigningRegion: region,- HostnameImmutable: true,- }, nil+// obsS3Options builds the S3 client options shared by the file service and the+// connectivity check.+func obsS3Options(endpoint, region, accessKey, secretAccessKey string) s3.Options {+ return s3.Options{+ Region: region,+ BaseEndpoint: aws.String(endpoint),+ Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretAccessKey, ""),+ // OBS is S3-compatible but commonly rejects the SDK's default+ // trailing checksum negotiation; align with the S3 driver (s3.go).+ RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,+ UsePathStyle: obsUsePathStyle(endpoint),+ HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),+ }}func NewObsFileService(@@ -48,13 +66,7 @@ func NewObsFileService(return nil, fmt.Errorf("unsafe OBS endpoint: %w", err)}- client := s3.New(s3.Options{- Region: region,- EndpointResolver: &obsEndpointResolver{url: endpoint},- Credentials: credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, ""),- UsePathStyle: true,- HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),- })+ client := s3.New(obsS3Options(endpoint, region, accessKeyID, secretAccessKey))_, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: aws.String(bucketName),@@ -87,13 +99,7 @@ func CheckObsConnectivity(ctx context.Context, endpoint, region, accessKey, secrcheckCtx, cancel := context.WithTimeout(ctx, 10*time.Second)defer cancel()- client := s3.New(s3.Options{- Region: region,- EndpointResolver: &obsEndpointResolver{url: endpoint},- Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),- UsePathStyle: true,- HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),- })+ client := s3.New(obsS3Options(endpoint, region, accessKey, secretKey))_, err := client.HeadBucket(checkCtx, &s3.HeadBucketInput{Bucket: aws.String(bucketName),@@ -236,7 +242,19 @@ func (s *obsFileService) GetFileURL(ctx context.Context, filePath string) (strinreturn s.proxyDomain + "/" + strings.TrimPrefix(objectKey, "/"), nil}- return fmt.Sprintf("%s/%s/%s", s.endpoint, s.bucketName, strings.TrimPrefix(objectKey, "/")), nil+ // Mirror the client addressing so generated URLs stay reachable: OBS+ // rejects path-style on domain endpoints (issue #3269), so serve+ // {bucket}.{endpoint-host}/{key}; IP-literal endpoints keep path-style.+ if obsUsePathStyle(s.endpoint) {+ return fmt.Sprintf("%s/%s/%s", s.endpoint, s.bucketName, strings.TrimPrefix(objectKey, "/")), nil+ }+ endpointURL, err := url.Parse(s.endpoint)+ if err != nil {+ return "", fmt.Errorf("invalid OBS endpoint %q: %w", s.endpoint, err)+ }+ endpointURL.Host = s.bucketName + "." + endpointURL.Host+ endpointURL.Path = "/" + strings.TrimPrefix(objectKey, "/")+ return endpointURL.String(), nil}// CopyFile copies an existing OBS object to a new knowledge-owned object using adiff --git a/internal/application/service/file/obs_test.go b/internal/application/service/file/obs_test.gonew file mode 100644index 000000000..313bbba9c--- /dev/null+++ b/internal/application/service/file/obs_test.go@@ -0,0 +1,198 @@+package file++import (+ "context"+ "io"+ "net/http"+ "strings"+ "testing"++ "github.com/aws/aws-sdk-go-v2/aws"+ "github.com/aws/aws-sdk-go-v2/service/s3"+)++// capturingRoundTripper records the first HTTP request URL the S3 client+// would send, without dialing the network. That is the invariant #3269+// actually cares about: PutObject must hit {bucket}.{host}/{key}, not+// {host}/{bucket}/{key}. Checking UsePathStyle alone is not enough — PR+// #3272 set that flag while HostnameImmutable still forced path-style.+type capturingRoundTripper struct {+ host string+ path string+}++func (c *capturingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {+ if c.host == "" {+ c.host = req.URL.Host+ c.path = req.URL.Path+ }+ return &http.Response{+ StatusCode: http.StatusOK,+ Header: http.Header{"Etag": []string{`"test"`}},+ Body: io.NopCloser(strings.NewReader("")),+ Request: req,+ Proto: "HTTP/1.1",+ ProtoMajor: 1,+ ProtoMinor: 1,+ }, nil+}++func putObjectWithCapture(t *testing.T, endpoint, region, bucket, key string) (host, path string) {+ t.Helper()+ trip := &capturingRoundTripper{}+ opts := obsS3Options(endpoint, region, "ak", "sk")+ opts.HTTPClient = &http.Client{Transport: trip}+ opts.Retryer = aws.NopRetryer{}+ client := s3.New(opts)+ _, err := client.PutObject(context.Background(), &s3.PutObjectInput{+ Bucket: aws.String(bucket),+ Key: aws.String(key),+ Body: strings.NewReader("hello"),+ })+ if err != nil {+ t.Fatalf("PutObject: %v", err)+ }+ if trip.host == "" {+ t.Fatal("PutObject did not issue an HTTP request")+ }+ return trip.host, trip.path+}++func TestObsS3OptionsUseVirtualHostedStyleForDomainEndpoints(t *testing.T) {+ const endpoint = "https://obs.cn-south-1.myhuaweicloud.com"+ opts := obsS3Options(endpoint, "cn-south-1", "ak", "sk")++ if opts.UsePathStyle {+ t.Fatal("domain endpoint must use virtual-hosted addressing (issue #3269)")+ }+ if opts.BaseEndpoint == nil || *opts.BaseEndpoint != endpoint {+ t.Fatalf("BaseEndpoint = %v, want %q", opts.BaseEndpoint, endpoint)+ }+ if opts.RequestChecksumCalculation != aws.RequestChecksumCalculationWhenRequired {+ t.Fatalf("RequestChecksumCalculation = %v, want WhenRequired", opts.RequestChecksumCalculation)+ }+ //nolint:staticcheck // v1 resolver must stay unset; HostnameImmutable pinned path-style+ if opts.EndpointResolver != nil {+ t.Fatal("deprecated custom resolver must stay dropped")+ }+ if opts.HTTPClient == nil {+ t.Fatal("SSRF-safe HTTP client must be configured")+ }+}++func TestObsS3OptionsKeepPathStyleForIPLiteralEndpoints(t *testing.T) {+ for _, endpoint := range []string{+ "http://192.168.1.10:9000",+ "https://[2001:db8::1]:443",+ "not a url",+ } {+ if opts := obsS3Options(endpoint, "region", "ak", "sk"); !opts.UsePathStyle {+ t.Fatalf("endpoint %q: expected defensive path-style fallback", endpoint)+ }+ }+}++func TestObsPutObjectUsesVirtualHostedURL(t *testing.T) {+ host, path := putObjectWithCapture(t,+ "https://obs.cn-south-1.myhuaweicloud.com",+ "cn-south-1",+ "my-bucket",+ "42/kb/file.pdf",+ )+ wantHost := "my-bucket.obs.cn-south-1.myhuaweicloud.com"+ if host != wantHost {+ t.Fatalf("PutObject host = %q, want virtual-hosted %q", host, wantHost)+ }+ if path != "/42/kb/file.pdf" {+ t.Fatalf("PutObject path = %q, want /42/kb/file.pdf", path)+ }+}++func TestObsPutObjectKeepsPathStyleForIPLiteralEndpoints(t *testing.T) {+ host, path := putObjectWithCapture(t,+ "http://192.168.1.10:9000",+ "region",+ "my-bucket",+ "key.bin",+ )+ if host != "192.168.1.10:9000" {+ t.Fatalf("PutObject host = %q, want 192.168.1.10:9000", host)+ }+ if path != "/my-bucket/key.bin" {+ t.Fatalf("PutObject path = %q, want /my-bucket/key.bin", path)+ }+}++func TestObsGetFileURLVirtualHostedStyle(t *testing.T) {+ s := &obsFileService{+ bucketName: "my-bucket",+ endpoint: "https://obs.cn-south-1.myhuaweicloud.com",+ }++ got, err := s.GetFileURL(context.Background(), "obs://my-bucket/42/kb/file.pdf")+ if err != nil {+ t.Fatal(err)+ }+ want := "https://my-bucket.obs.cn-south-1.myhuaweicloud.com/42/kb/file.pdf"+ if got != want {+ t.Fatalf("GetFileURL = %q, want %q", got, want)+ }+}++func TestObsGetFileURLPreservesEndpointPort(t *testing.T) {+ s := &obsFileService{+ bucketName: "my-bucket",+ endpoint: "https://obs.example.com:8443",+ }++ got, err := s.GetFileURL(context.Background(), "obs://my-bucket/key.bin")+ if err != nil {+ t.Fatal(err)+ }+ want := "https://my-bucket.obs.example.com:8443/key.bin"+ if got != want {+ t.Fatalf("GetFileURL = %q, want %q", got, want)+ }+}++func TestObsGetFileURLKeepsPathStyleForIPEndpoints(t *testing.T) {+ s := &obsFileService{+ bucketName: "my-bucket",+ endpoint: "http://192.168.1.10:9000",+ }++ got, err := s.GetFileURL(context.Background(), "obs://my-bucket/key.bin")+ if err != nil {+ t.Fatal(err)+ }+ want := "http://192.168.1.10:9000/my-bucket/key.bin"+ if got != want {+ t.Fatalf("GetFileURL = %q, want %q", got, want)+ }+}++func TestObsGetFileURLProxyDomainAndHTTPPassthrough(t *testing.T) {+ proxy := &obsFileService{+ bucketName: "my-bucket",+ proxyDomain: "https://cdn.example.com",+ }+ got, err := proxy.GetFileURL(context.Background(), "https://cdn.example.com/42/kb/file.pdf")+ if err != nil {+ t.Fatal(err)+ }+ if got != "https://cdn.example.com/42/kb/file.pdf" {+ t.Fatalf("proxy GetFileURL = %q", got)+ }++ s := &obsFileService{+ bucketName: "my-bucket",+ endpoint: "https://obs.cn-south-1.myhuaweicloud.com",+ }+ got, err = s.GetFileURL(context.Background(), "https://elsewhere.example.com/file")+ if err != nil {+ t.Fatal(err)+ }+ if got != "https://elsewhere.example.com/file" {+ t.Fatalf("http(s) passthrough = %q", got)+ }+}