Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

gcs: size the upload buffer from the entry instead of reserving the default 16 MiB chunk per writer (#11854)

jsasf8cfea0dcfed
diff --git a/weed/remote_storage/gcs/gcs_storage_client.go b/weed/remote_storage/gcs/gcs_storage_client.go
index c76881e988..20de17a645 100644
--- a/weed/remote_storage/gcs/gcs_storage_client.go
+++ b/weed/remote_storage/gcs/gcs_storage_client.go
@@ -14,6 +14,7 @@ import (
"time"
"cloud.google.com/go/storage"
+ "github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
@@ -348,6 +349,16 @@ func (gcs *gcsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocatio
wc.ContentType = entry.Attributes.Mime
}
wc.ContentEncoding = remote_storage.EntryContentEncoding(entry)
+ // A Writer reserves a buffer of ChunkSize bytes (16 MiB by default) so a
+ // failed request can be retried. An object below that size takes a buffer
+ // of its own size plus one 256 KiB unit, the SDK's granularity: a burst of
+ // small uploads in flight does not hold 16 MiB each, every upload keeps
+ // its retry buffer, and the object stays below ChunkSize so it goes in one
+ // request rather than a resumable session. The size counts the chunks,
+ // not only the attribute, which some write paths leave at zero.
+ if size := filer.FileSize(entry); size < uint64(googleapi.DefaultUploadChunkSize) {
+ wc.ChunkSize = int((size/googleapi.MinUploadChunkSize + 1) * googleapi.MinUploadChunkSize)
+ }
if _, err = io.Copy(wc, reader); err != nil {
return nil, fmt.Errorf("upload to gcs %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
}