Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(storage): let object storage transfers outlive 30 seconds (#3368)

Captaina5bdf82a02f8
diff --git a/internal/application/service/file/cos.go b/internal/application/service/file/cos.go
index 5bcc0b0e5..4ffaa3346 100644
--- a/internal/application/service/file/cos.go
+++ b/internal/application/service/file/cos.go
@@ -33,17 +33,15 @@ type cosFileService struct {
const cosScheme = "cos://"
func newCOSHTTPClient(secretID, secretKey string) *http.Client {
- httpConfig := utils.DefaultSSRFSafeHTTPClientConfig()
- client := utils.NewSSRFSafeHTTPClient(httpConfig)
+ client := objectStorageHTTPClient()
// COS GetCredential expects AuthorizationTransport to remain the outermost
- // transport. Put the per-request SSRF guard immediately underneath it while
- // retaining the safe client's redirect policy.
+ // transport. The per-request SSRF guard the client came with, over the
+ // bounded transport, goes immediately underneath it; the client keeps its
+ // redirect policy.
client.Transport = &cos.AuthorizationTransport{
SecretID: secretID,
SecretKey: secretKey,
- Transport: &utils.SSRFValidatingRoundTripper{
- Base: utils.NewSSRFSafeTransport(httpConfig),
- },
+ Transport: client.Transport,
}
return client
}
@@ -120,6 +118,8 @@ func (s *cosFileService) SaveFile(ctx context.Context,
return "", fmt.Errorf("failed to open file: %w", err)
}
defer src.Close()
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err = s.client.Object.Put(ctx, objectName, src, nil)
if err != nil {
return "", fmt.Errorf("failed to upload file to COS: %w", err)
@@ -136,11 +136,13 @@ func (s *cosFileService) GetFile(ctx context.Context, filePathUrl string) (io.Re
if err := utils.SafeObjectKey(objectName); err != nil {
return nil, fmt.Errorf("invalid file path: %w", err)
}
+ ctx, cancel := objectStorageTransferContext(ctx)
resp, err := s.client.Object.Get(ctx, objectName, nil)
if err != nil {
+ cancel()
return nil, fmt.Errorf("failed to get file from COS: %w", err)
}
- return resp.Body, nil
+ return objectStorageBoundReader(resp.Body, cancel), nil
}
// DeleteFile removes a file from COS storage
@@ -152,6 +154,8 @@ func (s *cosFileService) DeleteFile(ctx context.Context, filePath string) error
if err := utils.SafeObjectKey(objectName); err != nil {
return fmt.Errorf("invalid file path: %w", err)
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err = s.client.Object.Delete(ctx, objectName)
if err != nil {
return fmt.Errorf("failed to delete file: %w", err)
@@ -209,6 +213,9 @@ func (s *cosFileService) CopyFile(ctx context.Context,
destKey := fmt.Sprintf("%s/%d/%s/%s%s", s.cosPathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
// sourceURL is the host + object key WITHOUT a scheme, per the COS SDK contract.
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+
sourceURL := fmt.Sprintf("%s.cos.%s.myqcloud.com/%s", s.bucketName, s.region, srcObjectKey)
_, _, err = s.client.Object.Copy(ctx, destKey, sourceURL, nil)
if err != nil {
@@ -229,9 +236,11 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
return "", fmt.Errorf("invalid file name: %w", err)
}
ext := filepath.Ext(safeName)
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
reader := bytes.NewReader(data)
- // 如果请求写入临时桶且临时桶已配置
+ // ????????????????
if temp && s.tempClient != nil {
objectName := fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
_, err := s.tempClient.Object.Put(ctx, objectName, reader, nil)
@@ -242,7 +251,7 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
return fmt.Sprintf("%s%s", s.tempBucketURL, objectName), nil
}
- // 写入主桶
+ // ????
objectName := fmt.Sprintf("%s/%d/exports/%s%s", s.cosPathPrefix, tenantID, uuid.New().String(), ext)
_, err = s.client.Object.Put(ctx, objectName, reader, nil)
if err != nil {
@@ -254,7 +263,7 @@ func (s *cosFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
// GetFileURL returns a presigned download URL for the file
func (s *cosFileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
- // 判断文件属于哪个桶
+ // ?????????
if s.tempClient != nil && strings.HasPrefix(filePath, s.tempBucketURL) {
objectName := strings.TrimPrefix(filePath, s.tempBucketURL)
if err := utils.SafeObjectKey(objectName); err != nil {
diff --git a/internal/application/service/file/ks3.go b/internal/application/service/file/ks3.go
index 8954fbcda..2fbe4cd8c 100644
--- a/internal/application/service/file/ks3.go
+++ b/internal/application/service/file/ks3.go
@@ -3,9 +3,11 @@ package file
import (
"bytes"
"context"
+ "errors"
"fmt"
"io"
"mime/multipart"
+ "net/http"
"path/filepath"
"strings"
"time"
@@ -15,6 +17,7 @@ import (
"github.com/Tencent/WeKnora/internal/utils"
"github.com/google/uuid"
ks3aws "github.com/ks3sdklib/aws-sdk-go/aws"
+ "github.com/ks3sdklib/aws-sdk-go/aws/awserr"
"github.com/ks3sdklib/aws-sdk-go/aws/credentials"
ks3s3 "github.com/ks3sdklib/aws-sdk-go/service/s3"
)
@@ -65,20 +68,26 @@ func newKS3Client(endpoint, region, accessKey, secretKey string) (*ks3s3.S3, err
S3ForcePathStyle: false, // KS3 uses virtual-hosted style
SignerVersion: "V2", // KS3 recommends V2 signing
MaxRetries: 3,
- HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
+ HTTPClient: objectStorageHTTPClient(),
})
return client, nil
}
func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error {
- _, err := client.HeadBucket(&ks3s3.HeadBucketInput{
+ headCtx, cancel := objectStorageSetupContext()
+ _, err := client.HeadBucketWithContext(headCtx, &ks3s3.HeadBucketInput{
Bucket: ks3aws.String(bucketName),
})
+ cancel()
if err == nil {
return nil
}
- // Bucket doesn't exist, try to create it
- _, createErr := client.CreateBucket(&ks3s3.CreateBucketInput{
+ if !isKS3BucketMissing(err) {
+ return fmt.Errorf("failed to check KS3 bucket %q: %w", bucketName, err)
+ }
+ createCtx, cancel := objectStorageSetupContext()
+ defer cancel()
+ _, createErr := client.CreateBucketWithContext(createCtx, &ks3s3.CreateBucketInput{
Bucket: ks3aws.String(bucketName),
})
if createErr != nil {
@@ -87,6 +96,30 @@ func ensureKS3Bucket(client *ks3s3.S3, bucketName string) error {
return nil
}
+func isKS3BucketMissing(err error) bool {
+ if err == nil {
+ return false
+ }
+ var rf awserr.RequestFailure
+ if errors.As(err, &rf) {
+ if rf.StatusCode() == http.StatusNotFound {
+ return true
+ }
+ switch rf.Code() {
+ case "NoSuchBucket", "NotFound":
+ return true
+ }
+ }
+ var ae awserr.Error
+ if errors.As(err, &ae) {
+ switch ae.Code() {
+ case "NoSuchBucket", "NotFound":
+ return true
+ }
+ }
+ return false
+}
+
// CheckKS3Connectivity tests KS3 connectivity using the provided credentials.
func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secretKey, bucketName string) error {
client, err := newKS3Client(endpoint, region, accessKey, secretKey)
@@ -94,20 +127,12 @@ func CheckKS3Connectivity(ctx context.Context, endpoint, region, accessKey, secr
return err
}
- done := make(chan error, 1)
- go func() {
- _, err := client.HeadBucket(&ks3s3.HeadBucketInput{
- Bucket: ks3aws.String(bucketName),
- })
- done <- err
- }()
-
- select {
- case <-ctx.Done():
- return ctx.Err()
- case err := <-done:
- return err
- }
+ checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+ _, err = client.HeadBucketWithContext(checkCtx, &ks3s3.HeadBucketInput{
+ Bucket: ks3aws.String(bucketName),
+ })
+ return err
}
func joinKS3Key(parts ...string) string {
@@ -161,7 +186,9 @@ func (s *ks3FileService) SaveFile(ctx context.Context, file *multipart.FileHeade
contentType = utils.GetContentTypeByExt(ext)
}
- _, err = s.client.PutObject(&ks3s3.PutObjectInput{
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+ _, err = s.client.PutObjectWithContext(ctx, &ks3s3.PutObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
Body: src,
@@ -182,7 +209,9 @@ func (s *ks3FileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
ext := filepath.Ext(safeName)
objectKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), "exports", uuid.New().String()+ext)
- _, err = s.client.PutObject(&ks3s3.PutObjectInput{
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+ _, err = s.client.PutObjectWithContext(ctx, &ks3s3.PutObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
Body: bytes.NewReader(data),
@@ -215,7 +244,9 @@ func (s *ks3FileService) CopyFile(ctx context.Context,
ext := filepath.Ext(srcPath)
destKey := joinKS3Key(s.pathPrefix, fmt.Sprintf("%d", tenantID), knowledgeID, uuid.New().String()+ext)
- _, err = s.client.CopyObject(&ks3s3.CopyObjectInput{
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+ _, err = s.client.CopyObjectWithContext(ctx, &ks3s3.CopyObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(destKey),
SourceBucket: ks3aws.String(s.bucketName),
@@ -239,15 +270,17 @@ func (s *ks3FileService) GetFile(ctx context.Context, filePath string) (io.ReadC
return nil, fmt.Errorf("invalid file path: %w", err)
}
- resp, err := s.client.GetObject(&ks3s3.GetObjectInput{
+ ctx, cancel := objectStorageTransferContext(ctx)
+ resp, err := s.client.GetObjectWithContext(ctx, &ks3s3.GetObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
})
if err != nil {
+ cancel()
return nil, fmt.Errorf("failed to get file from KS3: %w", err)
}
- return resp.Body, nil
+ return objectStorageBoundReader(resp.Body, cancel), nil
}
func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error {
@@ -259,7 +292,9 @@ func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error
return fmt.Errorf("invalid file path: %w", err)
}
- _, err = s.client.DeleteObject(&ks3s3.DeleteObjectInput{
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+ _, err = s.client.DeleteObjectWithContext(ctx, &ks3s3.DeleteObjectInput{
Bucket: ks3aws.String(s.bucketName),
Key: ks3aws.String(objectKey),
})
@@ -270,19 +305,12 @@ func (s *ks3FileService) DeleteFile(ctx context.Context, filePath string) error
}
func (s *ks3FileService) CheckConnectivity(ctx context.Context) error {
- done := make(chan error, 1)
- go func() {
- _, err := s.client.HeadBucket(&ks3s3.HeadBucketInput{
- Bucket: ks3aws.String(s.bucketName),
- })
- done <- err
- }()
- select {
- case <-ctx.Done():
- return ctx.Err()
- case err := <-done:
- return err
- }
+ checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+ _, err := s.client.HeadBucketWithContext(checkCtx, &ks3s3.HeadBucketInput{
+ Bucket: ks3aws.String(s.bucketName),
+ })
+ return err
}
func (s *ks3FileService) GetFileURL(ctx context.Context, filePath string) (string, error) {
diff --git a/internal/application/service/file/object_storage_http.go b/internal/application/service/file/object_storage_http.go
new file mode 100644
index 000000000..5c15b35d5
--- /dev/null
+++ b/internal/application/service/file/object_storage_http.go
@@ -0,0 +1,100 @@
+package file
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/utils"
+)
+
+const (
+ // objectStorageSetupTimeout bounds the bucket probe a file service runs
+ // when it is built. Services are rebuilt per resolution, so the probe
+ // runs before many transfers, not just at startup.
+ objectStorageSetupTimeout = 30 * time.Second
+
+ // objectStorageTransferTimeout is the fallback when the caller did not
+ // set a deadline (asynq workers and the main HTTP server both omit one).
+ // Long enough for a slow 40 MB COS PUT; short enough that a stalled body
+ // cannot pin a worker forever.
+ objectStorageTransferTimeout = 30 * time.Minute
+
+ // objectStorageResponseHeaderTimeout is longer than a typical GET/PUT
+ // header wait because CopyObject / Object.Copy are synchronous: the
+ // server does not send headers until the copy finishes. GET headers
+ // still arrive immediately, so a silent peer fails in this window
+ // rather than the full transfer budget.
+ objectStorageResponseHeaderTimeout = 10 * time.Minute
+)
+
+// objectStorageHTTPClientConfig is the SSRF-safe client config for the object
+// storage SDKs. The default config's Timeout is Go's whole-request timeout,
+// body included, which capped every transfer at what a link moves in
+// 30 seconds (#3306). The object storage clients bound connection setup on
+// their transport instead and leave the transfer to the request context.
+func objectStorageHTTPClientConfig() utils.SSRFSafeHTTPClientConfig {
+ config := utils.DefaultSSRFSafeHTTPClientConfig()
+ config.Timeout = 0
+ return config
+}
+
+// objectStorageTransport is the SSRF-safe transport with the setup bounds the
+// whole-request timeout used to provide: a peer that accepts the connection
+// and then goes silent fails within the header wait instead of holding the
+// request until its context ends. The header wait starts after the request
+// body is written, so it does not cap an upload.
+func objectStorageTransport() *http.Transport {
+ transport := utils.NewSSRFSafeTransport(objectStorageHTTPClientConfig())
+ transport.TLSHandshakeTimeout = 10 * time.Second
+ transport.ResponseHeaderTimeout = objectStorageResponseHeaderTimeout
+ transport.ExpectContinueTimeout = time.Second
+ transport.IdleConnTimeout = 90 * time.Second
+ return transport
+}
+
+// objectStorageHTTPClient is the client for the SDKs that take a whole
+// http.Client (S3, KS3, OBS, OSS). COS wraps objectStorageTransport in its
+// own signing transport.
+func objectStorageHTTPClient() *http.Client {
+ return utils.NewSSRFSafeHTTPClientWithTransport(objectStorageHTTPClientConfig(), objectStorageTransport())
+}
+
+// objectStorageSetupContext bounds one constructor probe (HeadBucket or
+// CreateBucket). Callers must not reuse the same context for both: a slow
+// Head that then reports "missing" would leave Create with no time left.
+func objectStorageSetupContext() (context.Context, context.CancelFunc) {
+ return context.WithTimeout(context.Background(), objectStorageSetupTimeout)
+}
+
+// objectStorageTransferContext adds a fallback deadline only when the caller
+// did not set one, matching models/chat.withLLMTimeout.
+func objectStorageTransferContext(ctx context.Context) (context.Context, context.CancelFunc) {
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ if _, ok := ctx.Deadline(); ok {
+ return ctx, func() {}
+ }
+ return context.WithTimeout(ctx, objectStorageTransferTimeout)
+}
+
+type objectStorageBoundReadCloser struct {
+ io.ReadCloser
+ cancel context.CancelFunc
+}
+
+func objectStorageBoundReader(body io.ReadCloser, cancel context.CancelFunc) io.ReadCloser {
+ if body == nil {
+ cancel()
+ return nil
+ }
+ return &objectStorageBoundReadCloser{ReadCloser: body, cancel: cancel}
+}
+
+func (r *objectStorageBoundReadCloser) Close() error {
+ err := r.ReadCloser.Close()
+ r.cancel()
+ return err
+}
diff --git a/internal/application/service/file/object_storage_http_test.go b/internal/application/service/file/object_storage_http_test.go
new file mode 100644
index 000000000..eeb557251
--- /dev/null
+++ b/internal/application/service/file/object_storage_http_test.go
@@ -0,0 +1,136 @@
+package file
+
+import (
+ "context"
+ "io"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/ks3sdklib/aws-sdk-go/aws/awserr"
+ "github.com/stretchr/testify/require"
+ "github.com/tencentyun/cos-go-sdk-v5"
+
+ "github.com/Tencent/WeKnora/internal/utils"
+)
+
+// The object storage clients must not carry Go's whole-request timeout: it
+// bounds the body transfer too, so it capped every upload and download at
+// what a link can move in 30 seconds (#3306).
+func TestObjectStorageHTTPClientConfigDropsOnlyTheWholeRequestTimeout(t *testing.T) {
+ want := utils.DefaultSSRFSafeHTTPClientConfig()
+ want.Timeout = 0
+
+ require.Equal(t, want, objectStorageHTTPClientConfig())
+}
+
+// Without the whole-request timeout, connection setup is bounded on the
+// transport instead; a silent peer must still fail in bounded time.
+func TestObjectStorageTransportBoundsConnectionSetup(t *testing.T) {
+ transport := objectStorageTransport()
+
+ require.Equal(t, 10*time.Second, transport.TLSHandshakeTimeout)
+ require.Equal(t, objectStorageResponseHeaderTimeout, transport.ResponseHeaderTimeout)
+ require.Equal(t, time.Second, transport.ExpectContinueTimeout)
+ require.Equal(t, 90*time.Second, transport.IdleConnTimeout)
+ require.NotNil(t, transport.DialContext, "the SSRF-safe dialer must stay in place")
+}
+
+func TestObjectStorageHTTPClientKeepsTheSSRFGuard(t *testing.T) {
+ client := objectStorageHTTPClient()
+
+ require.Zero(t, client.Timeout)
+ _, ok := client.Transport.(*utils.SSRFValidatingRoundTripper)
+ require.True(t, ok)
+ require.NotNil(t, client.CheckRedirect)
+}
+
+func TestCOSHTTPClientHasNoWholeRequestTimeout(t *testing.T) {
+ client := newCOSHTTPClient("id", "key")
+
+ require.Zero(t, client.Timeout)
+ auth, ok := client.Transport.(*cos.AuthorizationTransport)
+ require.True(t, ok, "COS signing must stay the outermost transport")
+ guard, ok := auth.Transport.(*utils.SSRFValidatingRoundTripper)
+ require.True(t, ok, "the SSRF guard must stay underneath the signer")
+ base, ok := guard.Base.(*http.Transport)
+ require.True(t, ok)
+ require.Equal(t, objectStorageResponseHeaderTimeout, base.ResponseHeaderTimeout)
+}
+
+func TestOBSClientHasNoWholeRequestTimeout(t *testing.T) {
+ options := obsS3Options("obs.cn-north-4.myhuaweicloud.com", "cn-north-4", "ak", "sk")
+
+ client, ok := options.HTTPClient.(*http.Client)
+ require.True(t, ok)
+ require.Zero(t, client.Timeout)
+}
+
+func TestS3ClientHasNoWholeRequestTimeout(t *testing.T) {
+ svc, err := newS3Client("", "ak", "sk", "bucket", "us-east-1", "", false)
+ require.NoError(t, err)
+
+ client, ok := svc.client.Options().HTTPClient.(*http.Client)
+ require.True(t, ok)
+ require.Zero(t, client.Timeout)
+}
+
+func TestKS3ClientHasNoWholeRequestTimeout(t *testing.T) {
+ // The endpoint check resolves the host unless it is whitelisted; keep the
+ // test off the network.
+ t.Setenv("SSRF_WHITELIST", "ks3-cn-beijing.ksyuncs.com")
+ utils.ResetSSRFWhitelistForTest()
+ t.Cleanup(utils.ResetSSRFWhitelistForTest)
+ client, err := newKS3Client("https://ks3-cn-beijing.ksyuncs.com", "cn-beijing", "ak", "sk")
+ require.NoError(t, err)
+
+ require.Zero(t, client.Config.HTTPClient.Timeout)
+}
+
+func TestOSSClientHasNoWholeRequestTimeout(t *testing.T) {
+ t.Setenv("SSRF_WHITELIST", "oss-cn-hangzhou.aliyuncs.com")
+ utils.ResetSSRFWhitelistForTest()
+ t.Cleanup(utils.ResetSSRFWhitelistForTest)
+ client, err := newOSSClient("https://oss-cn-hangzhou.aliyuncs.com", "cn-hangzhou", "ak", "sk")
+ require.NoError(t, err)
+ require.NotNil(t, client)
+ require.Zero(t, objectStorageHTTPClient().Timeout)
+}
+
+func TestObjectStorageTransferContextAddsFallbackDeadline(t *testing.T) {
+ ctx, cancel := objectStorageTransferContext(context.Background())
+ defer cancel()
+
+ deadline, ok := ctx.Deadline()
+ require.True(t, ok)
+ remain := time.Until(deadline)
+ require.Greater(t, remain, 29*time.Minute)
+ require.Less(t, remain, 31*time.Minute)
+}
+
+func TestObjectStorageTransferContextKeepsCallerDeadline(t *testing.T) {
+ parent, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+
+ ctx, stop := objectStorageTransferContext(parent)
+ defer stop()
+ require.Equal(t, parent, ctx)
+}
+
+func TestObjectStorageBoundReaderCancelsOnClose(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ body := objectStorageBoundReader(io.NopCloser(strings.NewReader("x")), cancel)
+ require.NoError(t, body.Close())
+ require.ErrorIs(t, ctx.Err(), context.Canceled)
+}
+
+func TestKS3BucketMissingDetection(t *testing.T) {
+ require.True(t, isKS3BucketMissing(awserr.NewRequestFailure(
+ awserr.New("NoSuchBucket", "missing", nil), http.StatusNotFound, "req")))
+ require.True(t, isKS3BucketMissing(awserr.NewRequestFailure(
+ awserr.New("NotFound", "missing", nil), http.StatusNotFound, "req")))
+ require.False(t, isKS3BucketMissing(awserr.NewRequestFailure(
+ awserr.New("AccessDenied", "denied", nil), http.StatusForbidden, "req")))
+ require.False(t, isKS3BucketMissing(context.DeadlineExceeded))
+}
diff --git a/internal/application/service/file/obs.go b/internal/application/service/file/obs.go
index bc5df6e81..ecafa3a81 100644
--- a/internal/application/service/file/obs.go
+++ b/internal/application/service/file/obs.go
@@ -2,6 +2,7 @@ package file
import (
"context"
+ "errors"
"fmt"
"io"
"mime/multipart"
@@ -18,6 +19,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
+ s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/google/uuid"
)
@@ -54,7 +56,7 @@ func obsS3Options(endpoint, region, accessKey, secretAccessKey string) s3.Option
// trailing checksum negotiation; align with the S3 driver (s3.go).
RequestChecksumCalculation: aws.RequestChecksumCalculationWhenRequired,
UsePathStyle: obsUsePathStyle(endpoint),
- HTTPClient: utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()),
+ HTTPClient: objectStorageHTTPClient(),
}
}
@@ -68,15 +70,24 @@ func NewObsFileService(
client := s3.New(obsS3Options(endpoint, region, accessKeyID, secretAccessKey))
- _, err := client.HeadBucket(context.Background(), &s3.HeadBucketInput{
+ headCtx, cancel := objectStorageSetupContext()
+ _, err := client.HeadBucket(headCtx, &s3.HeadBucketInput{
Bucket: aws.String(bucketName),
})
+ cancel()
if err != nil {
- _, createErr := client.CreateBucket(context.Background(), &s3.CreateBucketInput{
- Bucket: aws.String(bucketName),
- })
- if createErr != nil {
- fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr)
+ var notFound *s3types.NotFound
+ if errors.As(err, &notFound) {
+ createCtx, createCancel := objectStorageSetupContext()
+ _, createErr := client.CreateBucket(createCtx, &s3.CreateBucketInput{
+ Bucket: aws.String(bucketName),
+ })
+ createCancel()
+ if createErr != nil {
+ fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, createErr)
+ }
+ } else {
+ fmt.Printf("Warning: bucket %s may not exist or cannot be created: %v\n", bucketName, err)
}
}
@@ -176,6 +187,8 @@ func (s *obsFileService) SaveFile(ctx context.Context,
contentType = "application/octet-stream"
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
@@ -200,15 +213,17 @@ func (s *obsFileService) GetFile(ctx context.Context, filePath string) (io.ReadC
return nil, err
}
+ ctx, cancel := objectStorageTransferContext(ctx)
output, err := s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
})
if err != nil {
+ cancel()
return nil, fmt.Errorf("failed to get file from OBS: %w", err)
}
- return output.Body, nil
+ return objectStorageBoundReader(output.Body, cancel), nil
}
func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error {
@@ -217,6 +232,8 @@ func (s *obsFileService) DeleteFile(ctx context.Context, filePath string) error
return err
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
@@ -285,6 +302,8 @@ func (s *obsFileService) CopyFile(ctx context.Context,
// CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded
// (url.PathEscape would turn them into %2F and break the bucket/key split).
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: aws.String(s.bucketName),
CopySource: aws.String(s.bucketName + "/" + srcKey),
@@ -323,6 +342,8 @@ func (s *obsFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
}
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err := s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectKey),
diff --git a/internal/application/service/file/oss.go b/internal/application/service/file/oss.go
index bde515e68..1f6d57f0b 100644
--- a/internal/application/service/file/oss.go
+++ b/internal/application/service/file/oss.go
@@ -43,14 +43,16 @@ func newOSSClient(endpoint, region, accessKey, secretKey string) (*oss.Client, e
WithCredentialsProvider(creds).
WithRegion(region).
WithEndpoint(endpoint).
- WithHttpClient(utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig()))
+ WithHttpClient(objectStorageHTTPClient())
return oss.NewClient(cfg), nil
}
// ossEnsureBucket checks if the bucket exists and creates it if missing.
func ossEnsureBucket(client *oss.Client, bucketName string) error {
- exists, err := client.IsBucketExist(context.Background(), bucketName)
+ headCtx, cancel := objectStorageSetupContext()
+ exists, err := client.IsBucketExist(headCtx, bucketName)
+ cancel()
if err != nil {
return fmt.Errorf("failed to check OSS bucket: %w", err)
}
@@ -58,7 +60,9 @@ func ossEnsureBucket(client *oss.Client, bucketName string) error {
return nil
}
- _, err = client.PutBucket(context.Background(), &oss.PutBucketRequest{
+ createCtx, cancel := objectStorageSetupContext()
+ defer cancel()
+ _, err = client.PutBucket(createCtx, &oss.PutBucketRequest{
Bucket: oss.Ptr(bucketName),
})
if err != nil {
@@ -123,7 +127,9 @@ func CheckOssConnectivity(ctx context.Context, endpoint, region, accessKey, secr
return err
}
- exists, err := client.IsBucketExist(ctx, bucketName)
+ checkCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+ exists, err := client.IsBucketExist(checkCtx, bucketName)
if err != nil {
return fmt.Errorf("failed to check OSS bucket: %w", err)
}
@@ -180,6 +186,9 @@ func (s *ossFileService) SaveFile(ctx context.Context,
contentType = utils.GetContentTypeByExt(ext)
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+
// Use Uploader for files > 10MB (auto multipart with concurrent uploads)
const multipartThreshold = 10 * 1024 * 1024
if file.Size > multipartThreshold {
@@ -234,6 +243,8 @@ func (s *ossFileService) SaveBytes(ctx context.Context, data []byte, tenantID ui
objectName = fmt.Sprintf("exports/%d/%s%s", tenantID, uuid.New().String(), ext)
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err = client.PutObject(ctx, &oss.PutObjectRequest{
Bucket: oss.Ptr(targetBucket),
Key: oss.Ptr(objectName),
@@ -264,6 +275,8 @@ func (s *ossFileService) CopyFile(ctx context.Context,
ext := filepath.Ext(srcPath)
destKey := fmt.Sprintf("%s%d/%s/%s%s", s.pathPrefix, tenantID, knowledgeID, uuid.New().String(), ext)
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err = s.client.CopyObject(ctx, &oss.CopyObjectRequest{
Bucket: oss.Ptr(s.bucketName),
Key: oss.Ptr(destKey),
@@ -296,15 +309,17 @@ func (s *ossFileService) GetFile(ctx context.Context, filePath string) (io.ReadC
client = s.client
}
+ ctx, cancel := objectStorageTransferContext(ctx)
resp, err := client.GetObject(ctx, &oss.GetObjectRequest{
Bucket: oss.Ptr(bucketName),
Key: oss.Ptr(objectName),
})
if err != nil {
+ cancel()
return nil, fmt.Errorf("failed to get file from OSS: %w", err)
}
- return resp.Body, nil
+ return objectStorageBoundReader(resp.Body, cancel), nil
}
// DeleteFile removes a file from OSS.
@@ -324,6 +339,8 @@ func (s *ossFileService) DeleteFile(ctx context.Context, filePath string) error
client = s.client
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
_, err = client.DeleteObject(ctx, &oss.DeleteObjectRequest{
Bucket: oss.Ptr(bucketName),
Key: oss.Ptr(objectName),
diff --git a/internal/application/service/file/s3.go b/internal/application/service/file/s3.go
index f43bb3b51..b405e2a24 100644
--- a/internal/application/service/file/s3.go
+++ b/internal/application/service/file/s3.go
@@ -58,7 +58,7 @@ func newS3Client(endpoint, accessKey, secretKey, bucketName, region, pathPrefix
// Create S3 client with custom endpoint if provided.
// For S3-compatible services (non-AWS), use path-style addressing
// (endpoint/bucket/key) instead of virtual-hosted style (bucket.endpoint/key).
- httpClient := utils.NewSSRFSafeHTTPClient(utils.DefaultSSRFSafeHTTPClientConfig())
+ httpClient := objectStorageHTTPClient()
var client *s3.Client
if endpoint != "" {
usePathStyle := forcePathStyle || !strings.Contains(endpoint, "amazonaws.com")
@@ -101,19 +101,9 @@ func NewS3FileService(endpoint,
if err != nil {
return nil, err
}
-
- // Check if bucket exists
- exists, err := svc.bucketExists(context.Background())
- if err != nil {
- return nil, fmt.Errorf("failed to check bucket: %w", err)
- }
-
- if !exists {
- if err = svc.createBucket(context.Background()); err != nil {
- return nil, fmt.Errorf("failed to create bucket: %w", err)
- }
+ if err := ensureS3Bucket(svc); err != nil {
+ return nil, err
}
-
return svc, nil
}
@@ -124,16 +114,28 @@ func NewS3FileServiceWithOptions(endpoint, accessKey, secretKey, bucketName, reg
if err != nil {
return nil, err
}
- exists, err := svc.bucketExists(context.Background())
+ if err := ensureS3Bucket(svc); err != nil {
+ return nil, err
+ }
+ return svc, nil
+}
+
+func ensureS3Bucket(svc *s3FileService) error {
+ headCtx, cancel := objectStorageSetupContext()
+ exists, err := svc.bucketExists(headCtx)
+ cancel()
if err != nil {
- return nil, fmt.Errorf("failed to check bucket: %w", err)
+ return fmt.Errorf("failed to check bucket: %w", err)
}
- if !exists {
- if err = svc.createBucket(context.Background()); err != nil {
- return nil, fmt.Errorf("failed to create bucket: %w", err)
- }
+ if exists {
+ return nil
}
- return svc, nil
+ createCtx, cancel := objectStorageSetupContext()
+ defer cancel()
+ if err := svc.createBucket(createCtx); err != nil {
+ return fmt.Errorf("failed to create bucket: %w", err)
+ }
+ return nil
}
// bucketExists checks if the bucket exists
@@ -238,6 +240,9 @@ func (s *s3FileService) SaveFile(ctx context.Context,
contentType = utils.GetContentTypeByExt(ext)
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+
// Upload file to S3
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(s.bucketName),
@@ -260,15 +265,17 @@ func (s *s3FileService) GetFile(ctx context.Context, filePath string) (io.ReadCl
return nil, err
}
+ ctx, cancel := objectStorageTransferContext(ctx)
resp, err := s.client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectName),
})
if err != nil {
+ cancel()
return nil, fmt.Errorf("failed to get file from S3: %w", err)
}
- return resp.Body, nil
+ return objectStorageBoundReader(resp.Body, cancel), nil
}
// DeleteFile deletes a file
@@ -278,6 +285,9 @@ func (s *s3FileService) DeleteFile(ctx context.Context, filePath string) error {
return err
}
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+
_, err = s.client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s.bucketName),
Key: aws.String(objectName),
@@ -306,6 +316,9 @@ func (s *s3FileService) CopyFile(ctx context.Context,
// CopySource is "bucket/key"; the '/' separators must NOT be percent-encoded
// (url.PathEscape would turn them into %2F and break the bucket/key split).
// srcKey is already validated by parseS3FilePath -> SafeObjectKey.
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+
_, err = s.client.CopyObject(ctx, &s3.CopyObjectInput{
Bucket: aws.String(s.bucketName),
CopySource: aws.String(s.bucketName + "/" + srcKey),
@@ -330,6 +343,9 @@ func (s *s3FileService) SaveBytes(ctx context.Context, data []byte, tenantID uin
ext := filepath.Ext(safeName)
objectName := fmt.Sprintf("%s%d/exports/%s%s", s.pathPrefix, tenantID, uuid.New().String(), ext)
+ ctx, cancel := objectStorageTransferContext(ctx)
+ defer cancel()
+
// Upload bytes to S3
reader := bytes.NewReader(data)
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
diff --git a/internal/container/container.go b/internal/container/container.go
index 16bd7df0a..a1d429a6f 100644
--- a/internal/container/container.go
+++ b/internal/container/container.go
@@ -582,7 +582,11 @@ func registerChatLocalImageResolver(
resourceCatalog interfaces.ResourceCatalog,
) {
chat.LocalImageResolver = func(storageURL string) ([]byte, bool) {
- ctx := context.Background()
+ // The object storage clients bound connection setup but leave the
+ // transfer to this context, so give it a deadline: a chat turn must
+ // not hang on one image whose download stalls.
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ defer cancel()
physicalPath, resource, err := resourceCatalog.ResolvePath(ctx, storageURL)
if err != nil {
return nil, false