SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
s3: read chunks from HTTPS volume servers with the global HTTP client (#11800)
Nabil Ragab6ab6926c94d0
diff --git a/weed/s3api/s3api_object_handlers.go b/weed/s3api/s3api_object_handlers.goindex f5e6b546cd..2078d4e69c 100644--- a/weed/s3api/s3api_object_handlers.go+++ b/weed/s3api/s3api_object_handlers.go@@ -1486,16 +1486,6 @@ func (s3a *S3ApiServer) streamRangeToClient(w io.Writer, r *http.Request, readerreturn cw.written, nil}-// Shared HTTP client for volume server requests (connection pooling)-var volumeServerHTTPClient = &http.Client{- Timeout: 5 * time.Minute,- Transport: &http.Transport{- MaxIdleConns: 100,- MaxIdleConnsPerHost: 10,- IdleConnTimeout: 90 * time.Second,- },-}-// createLookupFileIdFunction creates a reusable lookup function for resolving volume URLs// Uses FilerClient's vidMap cache to eliminate per-chunk gRPC overheadfunc (s3a *S3ApiServer) createLookupFileIdFunction() func(context.Context, string) ([]string, error) {@@ -2198,8 +2188,12 @@ func (s3a *S3ApiServer) readChunkRange(ctx context.Context, fileId string, offsereturn nil, fmt.Errorf("failed to lookup chunk %s: %w", fileId, err)}- // Use the first URL (already contains complete URL with fileId)- chunkUrl := urlStrings[0]+ // Use the first URL (already contains complete URL with fileId). The lookup returns http:// URLs; when+ // [https.client] is configured the volume servers only speak TLS, so take the global client's scheme.+ chunkUrl, err := util_http.NormalizeUrl(urlStrings[0])+ if err != nil {+ return nil, fmt.Errorf("failed to normalize chunk URL %s: %w", fileId, err)+ }// Generate JWT for volume server authentication (uses config loaded once at startup)jwt := filer.JwtForVolumeServer(fileId)@@ -2220,8 +2214,9 @@ func (s3a *S3ApiServer) readChunkRange(ctx context.Context, fileId string, offsereq.Header.Set("Authorization", security.BearerPrefix+jwt)}- // Use shared HTTP client with connection pooling- resp, err := volumeServerHTTPClient.Do(req)+ // Use the global HTTP client: it carries the [https.client] TLS configuration (CA and client+ // certificate) that HTTPS volume servers require; a bare http.Client cannot reach them.+ resp, err := util_http.GetGlobalHttpClient().Do(req)if err != nil {return nil, fmt.Errorf("failed to fetch chunk: %w", err)}