Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

s3: option to disable bucket auto-creation on upload (#10759)

Chris Luf66d6ffc4afe
diff --git a/weed/command/filer.go b/weed/command/filer.go
index a15075abbd..a0a90b2518 100644
--- a/weed/command/filer.go
+++ b/weed/command/filer.go
@@ -145,6 +145,7 @@ func init() {
filerS3Options.metricsHttpIp = cmdFiler.Flag.String("s3.metricsIp", "", "metrics listen ip. If empty, default to same as -s3.ip.bind option.")
cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
+ filerS3Options.autoCreateBucket = cmdFiler.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
diff --git a/weed/command/mini.go b/weed/command/mini.go
index a02d40e7db..3dd5853ff5 100644
--- a/weed/command/mini.go
+++ b/weed/command/mini.go
@@ -389,6 +389,7 @@ var (
miniS3Config = cmdMini.Flag.String("s3.config", "", "path to the S3 config file")
miniIamConfig = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3")
miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
+ miniS3AutoCreateBucket = cmdMini.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
miniBucket = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.")
miniTableBucket = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_TABLE_BUCKET env var.")
)
@@ -518,6 +519,7 @@ func initMiniS3Flags() {
miniS3Options.iamConfig = miniIamConfig
miniS3Options.auditLogConfig = cmdMini.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
miniS3Options.allowDeleteBucketNotEmpty = miniS3AllowDeleteBucketNotEmpty
+ miniS3Options.autoCreateBucket = miniS3AutoCreateBucket
miniS3Options.externalUrl = cmdMini.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Used for S3 signature verification behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")
miniS3Options.defaultFileMode = cmdMini.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")
miniS3Options.cacheSizeMB = cmdMini.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")
diff --git a/weed/command/s3.go b/weed/command/s3.go
index e57d33de8a..6d10370719 100644
--- a/weed/command/s3.go
+++ b/weed/command/s3.go
@@ -59,6 +59,7 @@ type S3Options struct {
metricsHttpPort *int
metricsHttpIp *string
allowDeleteBucketNotEmpty *bool
+ autoCreateBucket *bool
auditLogConfig *string
localFilerSocket *string
dataCenter *string
@@ -104,6 +105,7 @@ func init() {
s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")
cmdS3.Flag.Bool("allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
+ s3StandaloneOptions.autoCreateBucket = cmdS3.Flag.Bool("autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")
s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 120, "connection idle seconds")
@@ -351,6 +353,7 @@ func (s3opt *S3Options) startS3Server() bool {
BucketsPath: filerBucketsPath,
GrpcDialOption: grpcDialOption,
AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
+ AutoCreateBucket: *s3opt.autoCreateBucket,
LocalFilerSocket: localFilerSocket,
DataCenter: *s3opt.dataCenter,
FilerGroup: filerGroup,
diff --git a/weed/command/server.go b/weed/command/server.go
index e89d26682d..a40f131756 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -175,6 +175,7 @@ func init() {
s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")
s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
+ s3Options.autoCreateBucket = cmdServer.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")
s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")
s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")
s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 120, "connection idle seconds")
diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go
index 2f4cae39dc..5e261ef93d 100644
--- a/weed/s3api/s3api_bucket_handlers.go
+++ b/weed/s3api/s3api_bucket_handlers.go
@@ -589,6 +589,9 @@ func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorC
// ErrAutoCreatePermissionDenied is returned when a user lacks permission to auto-create buckets
var ErrAutoCreatePermissionDenied = errors.New("permission denied - requires Admin permission")
+// ErrAutoCreateDisabled is returned when bucket auto-creation is disabled by configuration
+var ErrAutoCreateDisabled = errors.New("bucket auto-creation is disabled")
+
// ErrInvalidBucketName is returned when a bucket name doesn't meet S3 naming requirements
var ErrInvalidBucketName = errors.New("invalid bucket name")
@@ -690,6 +693,10 @@ func (s3a *S3ApiServer) autoCreateBucket(r *http.Request, bucket string) error {
return fmt.Errorf("auto-create bucket %s: %w", bucket, errors.Join(ErrInvalidBucketName, err))
}
+ if !s3a.option.AutoCreateBucket {
+ return fmt.Errorf("auto-create bucket %s: %w", bucket, ErrAutoCreateDisabled)
+ }
+
// Check if user has admin permissions
if !s3a.isUserAdmin(r) {
return fmt.Errorf("auto-create bucket %s: %w", bucket, ErrAutoCreatePermissionDenied)
@@ -758,7 +765,9 @@ func (s3a *S3ApiServer) handleAutoCreateBucket(w http.ResponseWriter, r *http.Re
if err := s3a.autoCreateBucket(r, bucket); err != nil {
glog.Warningf("%s: %v", handlerName, err)
// Check for specific errors to return appropriate S3 error codes
- if errors.Is(err, ErrInvalidBucketName) {
+ if errors.Is(err, ErrAutoCreateDisabled) {
+ s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
+ } else if errors.Is(err, ErrInvalidBucketName) {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidBucketName)
} else if errors.Is(err, ErrAutoCreatePermissionDenied) {
s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)
diff --git a/weed/s3api/s3api_bucket_handlers_misc_test.go b/weed/s3api/s3api_bucket_handlers_misc_test.go
index b25d8720e4..8f8081e907 100644
--- a/weed/s3api/s3api_bucket_handlers_misc_test.go
+++ b/weed/s3api/s3api_bucket_handlers_misc_test.go
@@ -221,3 +221,67 @@ func TestGetBucketLogging(t *testing.T) {
t.Fatalf("missing xmlns: %s", got)
}
}
+
+func TestHandleAutoCreateBucketDisabled(t *testing.T) {
+ s3a := &S3ApiServer{option: &S3ApiServerOption{}}
+ req := newBucketRequest(http.MethodPut, "test-bucket", "", "")
+ rec := httptest.NewRecorder()
+
+ if s3a.handleAutoCreateBucket(rec, req, "test-bucket", "PutObjectHandler") {
+ t.Fatal("expected auto-create to be rejected")
+ }
+ if rec.Code != http.StatusNotFound {
+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
+ }
+ if !strings.Contains(rec.Body.String(), "NoSuchBucket") {
+ t.Fatalf("body = %s, want NoSuchBucket", rec.Body.String())
+ }
+}
+
+func TestHandleAutoCreateBucketNonAdmin(t *testing.T) {
+ s3a := &S3ApiServer{option: &S3ApiServerOption{AutoCreateBucket: true}}
+ req := newBucketRequest(http.MethodPut, "test-bucket", "", "")
+ rec := httptest.NewRecorder()
+
+ if s3a.handleAutoCreateBucket(rec, req, "test-bucket", "PutObjectHandler") {
+ t.Fatal("expected auto-create to be rejected")
+ }
+ if rec.Code != http.StatusForbidden {
+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusForbidden)
+ }
+}
+
+func TestUploadMissingBucketAutoCreateDisabled(t *testing.T) {
+ cases := []struct {
+ name string
+ method string
+ object string
+ handler func(*S3ApiServer, http.ResponseWriter, *http.Request)
+ }{
+ {"put object", http.MethodPut, "/key", (*S3ApiServer).PutObjectHandler},
+ {"put directory marker", http.MethodPut, "/dir/", (*S3ApiServer).PutObjectHandler},
+ {"new multipart upload", http.MethodPost, "/key", (*S3ApiServer).NewMultipartUploadHandler},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ s3a := &S3ApiServer{
+ option: &S3ApiServerOption{},
+ iam: &IdentityAccessManagement{},
+ bucketConfigCache: NewBucketConfigCache(time.Minute),
+ }
+ s3a.bucketConfigCache.SetNegativeCache("missing")
+ req := httptest.NewRequest(tc.method, "/missing"+tc.object, strings.NewReader(""))
+ req = mux.SetURLVars(req, map[string]string{"bucket": "missing", "object": tc.object})
+ rec := httptest.NewRecorder()
+
+ tc.handler(s3a, rec, req)
+
+ if rec.Code != http.StatusNotFound {
+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
+ }
+ if !strings.Contains(rec.Body.String(), "NoSuchBucket") {
+ t.Fatalf("body = %s, want NoSuchBucket", rec.Body.String())
+ }
+ })
+ }
+}
diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go
index 95d6d9ed03..5f8652ac78 100644
--- a/weed/s3api/s3api_object_handlers_put.go
+++ b/weed/s3api/s3api_object_handlers_put.go
@@ -131,6 +131,17 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
objectContentType := r.Header.Get("Content-Type")
if strings.HasSuffix(object, "/") && r.ContentLength <= 1024 {
+ // The filer mkdir below would implicitly create a missing bucket, so gate it
+ // on the same auto-create policy as regular uploads.
+ if err := s3a.checkBucket(r, bucket); err == s3err.ErrNoSuchBucket {
+ if !s3a.handleAutoCreateBucket(w, r, bucket, "PutObjectHandler") {
+ return
+ }
+ } else if err != s3err.ErrNone {
+ s3err.WriteErrorResponse(w, r, err)
+ return
+ }
+
// Split the object into directory path and name
objectWithoutSlash := strings.TrimSuffix(object, "/")
dirName := path.Dir(objectWithoutSlash)
diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go
index 7b2e6c746f..7b09e08550 100644
--- a/weed/s3api/s3api_server.go
+++ b/weed/s3api/s3api_server.go
@@ -52,6 +52,7 @@ type S3ApiServerOption struct {
BucketsPath string
GrpcDialOption grpc.DialOption
AllowDeleteBucketNotEmpty bool
+ AutoCreateBucket bool // create the bucket on upload if it does not exist
LocalFilerSocket string
DataCenter string
FilerGroup string