SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
s3api: remove the duplicated listing retry helpers left by overlapping merges
Chris Lu8d8a25b1cf14
diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.goindex 9c2c9f6585..38b73111c8 100644--- a/weed/s3api/filer_multipart.go+++ b/weed/s3api/filer_multipart.go@@ -1037,13 +1037,7 @@ func (s3a *S3ApiServer) listMultipartUploads(input *s3.ListMultipartUploadsInputIsTruncated: aws.Bool(false),}- // A blip on the way to the filer, either on the ListEntries call or on the- // stream receives that follow it, used to fail the whole listing; replay it- // a bounded number of times before giving up.- uploadsFolder := s3a.genUploadsFolder(*input.Bucket)- entries, _, err := listWithRetry(uploadsFolder, func() ([]*filer_pb.Entry, bool, error) {- return s3a.list(uploadsFolder, "", *input.UploadIdMarker, false, math.MaxInt32)- })+ entries, _, err := s3a.list(s3a.genUploadsFolder(*input.Bucket), "", *input.UploadIdMarker, false, math.MaxInt32)if err != nil {// A missing .uploads folder normally lists as empty with no error; a// store that reports it as not-found still means an empty list.@@ -1110,10 +1104,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListPStorageClass: aws.String("STANDARD"),}- partsFolder := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId- entries, isLast, err := listWithRetry(partsFolder, func() ([]*filer_pb.Entry, bool, error) {- return s3a.list(partsFolder, "", fmt.Sprintf("%04d%s", *input.PartNumberMarker, multipartExt), false, uint32(*input.MaxParts))- })+ entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d%s", *input.PartNumberMarker, multipartExt), false, uint32(*input.MaxParts))if err != nil {// A store that reports the missing upload directory as not-found means// the upload is gone (completed or aborted), not a store error.diff --git a/weed/s3api/filer_util.go b/weed/s3api/filer_util.goindex e15fb9f180..be4633e9e3 100644--- a/weed/s3api/filer_util.go+++ b/weed/s3api/filer_util.go@@ -83,52 +83,6 @@ func listWithRetry(parentDirectoryPath string, doList func() (entries []*filer_p}-// Bounds for replaying a listing that failed with a transient error. A listing-// is a read with no side effects and each attempt opens a new stream and-// collects into a fresh slice, so replaying it can neither duplicate nor drop-// entries. The bound keeps a filer that is genuinely down from stalling the S3-// request: at most two extra attempts and 300ms of added wait.-const (- listRetryAttempts = 3- listRetryInitialBackoff = 100 * time.Millisecond-)--// isRetryableListError reports whether a failed listing is worth replaying. A-// not-found answer is authoritative and every other non-transient error is a-// real failure, so both must reach the caller unchanged: the point is to-// survive a blip, not to hide a broken store.-func isRetryableListError(err error) bool {- if err == nil || isFilerNotFound(err) {- return false- }- if status.Code(err) == codes.Unavailable {- return true- }- // DoSeaweedListWithSnapshot wraps a failed ListEntries call with %v, which- // drops the gRPC status from the error chain, so for that path the message- // is all that is left to classify on.- return util.IsTransientError(err)-}--// listWithRetry replays doList while the filer answers with a transient error.-// Both failure points reported for multipart listing, the ListEntries call-// itself and the stream.Recv that follows it, surface as a plain error out of-// filer_pb.List, so a single retry point above it covers both.-func listWithRetry(parentDirectoryPath string, doList func() (entries []*filer_pb.Entry, isLast bool, err error)) (entries []*filer_pb.Entry, isLast bool, err error) {-- backoff := listRetryInitialBackoff- for attempt := 1; ; attempt++ {- entries, isLast, err = doList()- if err == nil || attempt >= listRetryAttempts || !isRetryableListError(err) {- return entries, isLast, err- }- glog.V(1).Infof("list %s attempt %d/%d hit a transient error, retrying in %v: %v", parentDirectoryPath, attempt, listRetryAttempts, backoff, err)- time.Sleep(backoff)- backoff *= 2- }--}-func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {return s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {