SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
refactor: centralize genUploadUrl in UploadOption (#10164)
MorezMartinbdcc3154ed38
diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.goindex f4a712f8f9..4b814116d4 100644--- a/weed/command/filer_copy.go+++ b/weed/command/filer_copy.go@@ -371,9 +371,6 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) errMimeType: mimeType,PairMap: nil,},- func(host, fileId string) string {- return fmt.Sprintf("http://%s/%s", host, fileId)- },util.NewBytesReader(data),)if flushErr != nil {@@ -456,9 +453,6 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,MimeType: "",PairMap: nil,},- func(host, fileId string) string {- return fmt.Sprintf("http://%s/%s", host, fileId)- },io.NewSectionReader(f, i*chunkSize, chunkSize),)@@ -576,9 +570,6 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, offMimeType: "",PairMap: nil,},- func(host, fileId string) string {- return fmt.Sprintf("http://%s/%s", host, fileId)- },reader,)diff --git a/weed/filer/gateway_upload.go b/weed/filer/gateway_upload.goindex 507b5efa6a..7b6da4a25c 100644--- a/weed/filer/gateway_upload.go+++ b/weed/filer/gateway_upload.go@@ -17,7 +17,6 @@ type GatewayChunkUploader interface {filerClient filer_pb.FilerClient,assignRequest *filer_pb.AssignVolumeRequest,uploadOption *operation.UploadOption,- genFileUrlFn func(host, fileId string) string,reader io.Reader,) (fileId string, uploadResult *operation.UploadResult, err error, data []byte)}@@ -120,11 +119,8 @@ func SaveGatewayDataAsChunk(req GatewayChunkUploadRequest) (*filer_pb.FileChunk,PairMap: req.PairMap,}- genFileUrlFn := func(host, fileId string) string {- if req.VolumeServerAccess == "filerProxy" && req.FilerHTTPAddress != "" {- return fmt.Sprintf("http://%s/?proxyChunkId=%s", req.FilerHTTPAddress, fileId)- }- return fmt.Sprintf("http://%s/%s", host, fileId)+ if req.VolumeServerAccess == "filerProxy" && req.FilerHTTPAddress != "" {+ uploadOption.GenUploadUrl = operation.GenUploadUrlProxy(req.FilerHTTPAddress)}assignRequest := &filer_pb.AssignVolumeRequest{@@ -141,7 +137,6 @@ func SaveGatewayDataAsChunk(req GatewayChunkUploadRequest) (*filer_pb.FileChunk,req.FilerClient,assignRequest,uploadOption,- genFileUrlFn,req.Reader,)if uploadErr != nil {diff --git a/weed/mount/weedfs_write.go b/weed/mount/weedfs_write.goindex dfe5a6e484..8bb6a41572 100644--- a/weed/mount/weedfs_write.go+++ b/weed/mount/weedfs_write.go@@ -35,12 +35,11 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFunPairMap: nil,WantMd5: true,}- genFileUrlFn := func(host, fileId string) string {- fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)- if wfs.option.VolumeServerAccess == "filerProxy" {- fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId)+ if wfs.option.VolumeServerAccess == "filerProxy" {+ // getCurrentFiler() can change on failover, so read it per attempt.+ uploadOption.GenUploadUrl = func(host, fileId string) string {+ return fmt.Sprintf("http://%s/?proxyChunkId=%s", wfs.getCurrentFiler(), fileId)}- return fileUrl}fileId, uploadResult, err, data := uploader.UploadWithRetry(@@ -54,7 +53,7 @@ func (wfs *WFS) saveDataAsChunk(fullPath util.FullPath) filer.SaveDataAsChunkFunDataCenter: wfs.option.DataCenter,Path: assignPath,},- uploadOption, genFileUrlFn, reader,+ uploadOption, reader,)if err != nil {diff --git a/weed/mq/broker/broker_write.go b/weed/mq/broker/broker_write.goindex bdb72a770d..4b91e49e01 100644--- a/weed/mq/broker/broker_write.go+++ b/weed/mq/broker/broker_write.go@@ -161,27 +161,26 @@ func (b *MessageQueueBroker) assignAndUpload(targetFile string, data []byte) (fireturn}+ uploadOption := &operation.UploadOption{+ Cipher: b.option.Cipher,+ }+ if b.option.VolumeServerAccess == "filerProxy" {+ // b.currentFiler can change on failover, so read it per attempt.+ uploadOption.GenUploadUrl = func(host, fileId string) string {+ return fmt.Sprintf("http://%s/?proxyChunkId=%s", b.currentFiler, fileId)+ }+ }+fileId, uploadResult, err, _ = uploader.UploadWithRetry(b,&filer_pb.AssignVolumeRequest{Count: 1,Replication: b.option.DefaultReplication,Collection: "topics",- // TtlSec: wfs.option.TtlSec,- // DiskType: string(wfs.option.DiskType),- DataCenter: b.option.DataCenter,- Path: targetFile,- },- &operation.UploadOption{- Cipher: b.option.Cipher,- },- func(host, fileId string) string {- fileUrl := fmt.Sprintf("http://%s/%s", host, fileId)- if b.option.VolumeServerAccess == "filerProxy" {- fileUrl = fmt.Sprintf("http://%s/?proxyChunkId=%s", b.currentFiler, fileId)- }- return fileUrl+ DataCenter: b.option.DataCenter,+ Path: targetFile,},+ uploadOption,reader,)returndiff --git a/weed/mq/logstore/log_to_parquet.go b/weed/mq/logstore/log_to_parquet.goindex bfd5ff10e6..067ddf1b09 100644--- a/weed/mq/logstore/log_to_parquet.go+++ b/weed/mq/logstore/log_to_parquet.go@@ -479,9 +479,6 @@ func saveParquetFileToPartitionDir(filerClient filer_pb.FilerClient, sourceFileMimeType: "application/vnd.apache.parquet",PairMap: nil,},- func(host, fileId string) string {- return fmt.Sprintf("http://%s/%s", host, fileId)- },io.NewSectionReader(sourceFile, i*chunkSize, chunkSize),)if err != nil {diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.goindex 493d1c2c96..946149ade2 100644--- a/weed/operation/upload_content.go+++ b/weed/operation/upload_content.go@@ -30,6 +30,21 @@ import (util_http_client "github.com/seaweedfs/seaweedfs/weed/util/http/client")+// GenUploadUrlProxy returns a function that builds a chunk upload URL via the+// filer proxy. Identical to the inline logic used by weed mount -filerProxy+// and weed filer gateway:+//+// - without filerProxy: "http://{host}/{fileId}"+// - with filerProxy: "http://{filerAddress}/?proxyChunkId={fileId}"+func GenUploadUrlProxy(filerAddress string) func(host, fileId string) string {+ return func(host, fileId string) string {+ if filerAddress == "" {+ return fmt.Sprintf("http://%s/%s", host, fileId)+ }+ return fmt.Sprintf("http://%s/?proxyChunkId=%s", filerAddress, fileId)+ }+}+type UploadOption struct {UploadUrl stringFilename string@@ -43,8 +58,9 @@ type UploadOption struct {Md5 stringWantMd5 bool // compute Content-MD5 from the data when Md5 is unset and the upload is not cipheredBytesBuffer *bytes.Buffer- SourceUrl string // optional: for logging when reading from a remote source- MaxAttempts int // <=0 uses the default+ SourceUrl string // optional: for logging when reading from a remote source+ MaxAttempts int // <=0 uses the default+ GenUploadUrl func(host, fileId string) string // if nil → fallback "http://{host}/{fileId}"}type UploadResult struct {@@ -153,7 +169,7 @@ func NewUploaderWithHttpClient(httpClient HTTPClient) *Uploader {}}-func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, err error), uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, data []byte) (fileId string, uploadResult *UploadResult, err error) {+func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, host string, auth security.EncodedJwt, err error), uploadOption *UploadOption, data []byte) (fileId string, uploadResult *UploadResult, err error) {doUploadFunc := func() error {var host stringvar auth security.EncodedJwt@@ -162,7 +178,11 @@ func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, horeturn err}- uploadOption.UploadUrl = genFileUrlFn(host, fileId)+ genUrl := uploadOption.GenUploadUrl+ if genUrl == nil {+ genUrl = func(host, fileId string) string { return fmt.Sprintf("http://%s/%s", host, fileId) }+ }+ uploadOption.UploadUrl = genUrl(host, fileId)uploadOption.Jwt = authuploadResult, err = uploader.retriedUploadData(context.Background(), data, uploadOption)@@ -183,7 +203,7 @@ func (uploader *Uploader) uploadWithRetryData(assignFn func() (fileId string, ho// UploadWithRetry will retry both assigning volume request and uploading content// The option parameter does not need to specify UploadUrl and Jwt, which will come from assigning volume.-func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.AssignVolumeRequest, uploadOption *UploadOption, genFileUrlFn func(host, fileId string) string, reader io.Reader) (fileId string, uploadResult *UploadResult, err error, data []byte) {+func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assignRequest *filer_pb.AssignVolumeRequest, uploadOption *UploadOption, reader io.Reader) (fileId string, uploadResult *UploadResult, err error, data []byte) {bytesReader, ok := reader.(*util.BytesReader)if ok {data = bytesReader.Bytes@@ -234,7 +254,7 @@ func (uploader *Uploader) UploadWithRetry(filerClient filer_pb.FilerClient, assierr = fmt.Errorf("filerGrpcAddress assign volume: %w", grpcAssignErr)}return- }, uploadOption, genFileUrlFn, data)+ }, uploadOption, data)return}diff --git a/weed/operation/upload_content_test.go b/weed/operation/upload_content_test.goindex bd1e10e508..2b91c00568 100644--- a/weed/operation/upload_content_test.go+++ b/weed/operation/upload_content_test.go@@ -107,9 +107,7 @@ func TestUploadWithRetryDataReassignsOnVolumeSizeExceeded(t *testing.T) {return "1,first", "volume-a", "", nil}return "2,second", "volume-b", "", nil- }, &UploadOption{Filename: "test.bin"}, func(host, fileId string) string {- return "http://" + host + "/" + fileId- }, []byte("abc"))+ }, &UploadOption{Filename: "test.bin"}, []byte("abc"))if err != nil {t.Fatalf("expected success after reassignment, got %v", err)@@ -210,7 +208,6 @@ func TestUploadWithRetryBoundsAssignVolume(t *testing.T) {_, _, err, _ := uploader.UploadWithRetry(client,&filer_pb.AssignVolumeRequest{Count: 1},&UploadOption{Filename: "test.bin"},- func(host, fileId string) string { return "http://" + host + "/" + fileId },bytes.NewReader([]byte("abc")),)done <- errdiff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.goindex 3f0742fbd6..4bfa354d31 100644--- a/weed/replication/sink/filersink/fetch_write.go+++ b/weed/replication/sink/filersink/fetch_write.go@@ -187,6 +187,17 @@ func (fs *FilerSink) uploadManifestChunk(path string, sourceMtimeNs int64, sourcretryName := fmt.Sprintf("replicate manifest chunk %s", sourceFileId)err = util.RetryUntil(retryName, func() error {+ uploadOption := &operation.UploadOption{+ Filename: "",+ Cipher: false,+ IsInputCompressed: false,+ MimeType: "application/octet-stream",+ PairMap: nil,+ RetryForever: false,+ }+ if fs.writeChunkByFiler {+ uploadOption.GenUploadUrl = operation.GenUploadUrlProxy(fs.address)+ }currentFileId, uploadResult, uploadErr, _ := uploader.UploadWithRetry(fs,&filer_pb.AssignVolumeRequest{@@ -198,17 +209,7 @@ func (fs *FilerSink) uploadManifestChunk(path string, sourceMtimeNs int64, sourcDiskType: fs.diskType,Path: path,},- &operation.UploadOption{- Filename: "",- Cipher: false,- IsInputCompressed: false,- MimeType: "application/octet-stream",- PairMap: nil,- RetryForever: false,- },- func(host, fileId string) string {- return fs.buildUploadUrl(host, fileId)- },+ uploadOption,bytes.NewReader(manifestData),)if uploadErr != nil {@@ -302,6 +303,18 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string,transferStatus.Status = "uploading"transferStatus.mu.Unlock()+ uploadOption := &operation.UploadOption{+ Filename: savedFilename,+ Cipher: false,+ IsInputCompressed: "gzip" == savedHeader.Get("Content-Encoding"),+ MimeType: savedHeader.Get("Content-Type"),+ PairMap: nil,+ RetryForever: false,+ SourceUrl: savedSourceUrl,+ }+ if fs.writeChunkByFiler {+ uploadOption.GenUploadUrl = operation.GenUploadUrlProxy(fs.address)+ }currentFileId, uploadResult, uploadErr, _ := uploader.UploadWithRetry(fs,&filer_pb.AssignVolumeRequest{@@ -313,20 +326,7 @@ func (fs *FilerSink) fetchAndWrite(sourceChunk *filer_pb.FileChunk, path string,DiskType: fs.diskType,Path: path,},- &operation.UploadOption{- Filename: savedFilename,- Cipher: false,- IsInputCompressed: "gzip" == savedHeader.Get("Content-Encoding"),- MimeType: savedHeader.Get("Content-Type"),- PairMap: nil,- RetryForever: false,- SourceUrl: savedSourceUrl,- },- func(host, fileId string) string {- fileUrl := fs.buildUploadUrl(host, fileId)- glog.V(4).Infof("replicating %s to %s header:%+v", savedFilename, fileUrl, savedHeader)- return fileUrl- },+ uploadOption,util.NewBytesReader(fullData),)if uploadErr != nil {@@ -435,13 +435,6 @@ func validateReplicatedReadSize(sourceChunk *filer_pb.FileChunk, readSize int) ereturn nil}-func (fs *FilerSink) buildUploadUrl(host, fileId string) string {- if fs.writeChunkByFiler {- return fmt.Sprintf("http://%s/?proxyChunkId=%s", fs.address, fileId)- }- return fmt.Sprintf("http://%s/%s", host, fileId)-}-// hasSourceNewerVersion reports whether the source's current entry for targetPath// has moved past sourceMtimeNs — gone, or a strictly-newer mtime — meaning the// version being replayed is stale. The lookup runs regardless of sourceMtimeNs sodiff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.goindex 72fbd02be8..62e65b0ef9 100644--- a/weed/server/filer_server_handlers_read.go+++ b/weed/server/filer_server_handlers_read.go@@ -254,5 +254,16 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)}func (fs *FilerServer) maybeGetVolumeReadJwtAuthorizationToken(fileId string) string {- return string(security.GenJwtForVolumeServer(fs.volumeGuard.ReadSigningKey(), fs.volumeGuard.ReadExpiresAfterSec(), fileId))+ // Generate a read JWT for volume server access. If the dedicated+ // read key (jwt.signing.read.key) is not configured, fall back to the+ // general signing key (jwt.signing.key) so the proxy can still+ // authenticate to volume servers that require JWT.+ key := fs.volumeGuard.ReadSigningKey()+ if len(key) == 0 {+ key = fs.volumeGuard.SigningKey()+ }+ if len(key) == 0 {+ return ""+ }+ return string(security.GenJwtForVolumeServer(key, fs.volumeGuard.ReadExpiresAfterSec(), fileId))}