SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
s3: option to disable bucket auto-creation on upload (#10759)
Chris Luf66d6ffc4afe
diff --git a/weed/command/filer.go b/weed/command/filer.goindex a15075abbd..a0a90b2518 100644--- a/weed/command/filer.go+++ b/weed/command/filer.go@@ -145,6 +145,7 @@ func init() {filerS3Options.metricsHttpIp = cmdFiler.Flag.String("s3.metricsIp", "", "metrics listen ip. If empty, default to same as -s3.ip.bind option.")cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")filerS3Options.allowDeleteBucketNotEmpty = cmdFiler.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")+ filerS3Options.autoCreateBucket = cmdFiler.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")filerS3Options.localSocket = cmdFiler.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")filerS3Options.tlsCACertificate = cmdFiler.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")filerS3Options.tlsVerifyClientCert = cmdFiler.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")diff --git a/weed/command/mini.go b/weed/command/mini.goindex a02d40e7db..3dd5853ff5 100644--- a/weed/command/mini.go+++ b/weed/command/mini.go@@ -389,6 +389,7 @@ var (miniS3Config = cmdMini.Flag.String("s3.config", "", "path to the S3 config file")miniIamConfig = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3")miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")+ miniS3AutoCreateBucket = cmdMini.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")miniBucket = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.")miniTableBucket = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_TABLE_BUCKET env var."))@@ -518,6 +519,7 @@ func initMiniS3Flags() {miniS3Options.iamConfig = miniIamConfigminiS3Options.auditLogConfig = cmdMini.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")miniS3Options.allowDeleteBucketNotEmpty = miniS3AllowDeleteBucketNotEmpty+ miniS3Options.autoCreateBucket = miniS3AutoCreateBucketminiS3Options.externalUrl = cmdMini.Flag.String("s3.externalUrl", "", "the external URL clients use to connect (e.g. https://api.example.com:9000). Used for S3 signature verification behind a reverse proxy. Falls back to S3_EXTERNAL_URL env var.")miniS3Options.defaultFileMode = cmdMini.Flag.String("s3.defaultFileMode", "", "default file mode for S3 uploaded objects, e.g. 0660, 0644, 0666")miniS3Options.cacheSizeMB = cmdMini.Flag.Int64("s3.cacheCapacityMB", 0, "in-memory chunk cache capacity in MB for S3 GETs shared across requests (0 disables)")diff --git a/weed/command/s3.go b/weed/command/s3.goindex e57d33de8a..6d10370719 100644--- a/weed/command/s3.go+++ b/weed/command/s3.go@@ -59,6 +59,7 @@ type S3Options struct {metricsHttpPort *intmetricsHttpIp *stringallowDeleteBucketNotEmpty *bool+ autoCreateBucket *boolauditLogConfig *stringlocalFilerSocket *stringdataCenter *string@@ -104,6 +105,7 @@ func init() {s3StandaloneOptions.metricsHttpIp = cmdS3.Flag.String("metricsIp", "", "metrics listen ip. If empty, default to same as -ip.bind option.")cmdS3.Flag.Bool("allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")+ s3StandaloneOptions.autoCreateBucket = cmdS3.Flag.Bool("autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")s3StandaloneOptions.localFilerSocket = cmdS3.Flag.String("localFilerSocket", "", "local filer socket path")s3StandaloneOptions.localSocket = cmdS3.Flag.String("localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")s3StandaloneOptions.idleTimeout = cmdS3.Flag.Int("idleTimeout", 120, "connection idle seconds")@@ -351,6 +353,7 @@ func (s3opt *S3Options) startS3Server() bool {BucketsPath: filerBucketsPath,GrpcDialOption: grpcDialOption,AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,+ AutoCreateBucket: *s3opt.autoCreateBucket,LocalFilerSocket: localFilerSocket,DataCenter: *s3opt.dataCenter,FilerGroup: filerGroup,diff --git a/weed/command/server.go b/weed/command/server.goindex e89d26682d..a40f131756 100644--- a/weed/command/server.go+++ b/weed/command/server.go@@ -175,6 +175,7 @@ func init() {s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "deprecated, ignored. Empty folder cleanup is now automatic.")s3Options.allowDeleteBucketNotEmpty = cmdServer.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")+ s3Options.autoCreateBucket = cmdServer.Flag.Bool("s3.autoCreateBucket", true, "create the bucket on upload if it does not exist, for admin identities only")s3Options.localSocket = cmdServer.Flag.String("s3.localSocket", "", "default to /tmp/seaweedfs-s3-<port>.sock")s3Options.bindIp = cmdServer.Flag.String("s3.ip.bind", "", "ip address to bind to. If empty, default to same as -ip.bind option.")s3Options.idleTimeout = cmdServer.Flag.Int("s3.idleTimeout", 120, "connection idle seconds")diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.goindex 2f4cae39dc..5e261ef93d 100644--- a/weed/s3api/s3api_bucket_handlers.go+++ b/weed/s3api/s3api_bucket_handlers.go@@ -589,6 +589,9 @@ func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorC// ErrAutoCreatePermissionDenied is returned when a user lacks permission to auto-create bucketsvar ErrAutoCreatePermissionDenied = errors.New("permission denied - requires Admin permission")+// ErrAutoCreateDisabled is returned when bucket auto-creation is disabled by configuration+var ErrAutoCreateDisabled = errors.New("bucket auto-creation is disabled")+// ErrInvalidBucketName is returned when a bucket name doesn't meet S3 naming requirementsvar ErrInvalidBucketName = errors.New("invalid bucket name")@@ -690,6 +693,10 @@ func (s3a *S3ApiServer) autoCreateBucket(r *http.Request, bucket string) error {return fmt.Errorf("auto-create bucket %s: %w", bucket, errors.Join(ErrInvalidBucketName, err))}+ if !s3a.option.AutoCreateBucket {+ return fmt.Errorf("auto-create bucket %s: %w", bucket, ErrAutoCreateDisabled)+ }+// Check if user has admin permissionsif !s3a.isUserAdmin(r) {return fmt.Errorf("auto-create bucket %s: %w", bucket, ErrAutoCreatePermissionDenied)@@ -758,7 +765,9 @@ func (s3a *S3ApiServer) handleAutoCreateBucket(w http.ResponseWriter, r *http.Reif err := s3a.autoCreateBucket(r, bucket); err != nil {glog.Warningf("%s: %v", handlerName, err)// Check for specific errors to return appropriate S3 error codes- if errors.Is(err, ErrInvalidBucketName) {+ if errors.Is(err, ErrAutoCreateDisabled) {+ s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)+ } else if errors.Is(err, ErrInvalidBucketName) {s3err.WriteErrorResponse(w, r, s3err.ErrInvalidBucketName)} else if errors.Is(err, ErrAutoCreatePermissionDenied) {s3err.WriteErrorResponse(w, r, s3err.ErrAccessDenied)diff --git a/weed/s3api/s3api_bucket_handlers_misc_test.go b/weed/s3api/s3api_bucket_handlers_misc_test.goindex b25d8720e4..8f8081e907 100644--- a/weed/s3api/s3api_bucket_handlers_misc_test.go+++ b/weed/s3api/s3api_bucket_handlers_misc_test.go@@ -221,3 +221,67 @@ func TestGetBucketLogging(t *testing.T) {t.Fatalf("missing xmlns: %s", got)}}++func TestHandleAutoCreateBucketDisabled(t *testing.T) {+ s3a := &S3ApiServer{option: &S3ApiServerOption{}}+ req := newBucketRequest(http.MethodPut, "test-bucket", "", "")+ rec := httptest.NewRecorder()++ if s3a.handleAutoCreateBucket(rec, req, "test-bucket", "PutObjectHandler") {+ t.Fatal("expected auto-create to be rejected")+ }+ if rec.Code != http.StatusNotFound {+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)+ }+ if !strings.Contains(rec.Body.String(), "NoSuchBucket") {+ t.Fatalf("body = %s, want NoSuchBucket", rec.Body.String())+ }+}++func TestHandleAutoCreateBucketNonAdmin(t *testing.T) {+ s3a := &S3ApiServer{option: &S3ApiServerOption{AutoCreateBucket: true}}+ req := newBucketRequest(http.MethodPut, "test-bucket", "", "")+ rec := httptest.NewRecorder()++ if s3a.handleAutoCreateBucket(rec, req, "test-bucket", "PutObjectHandler") {+ t.Fatal("expected auto-create to be rejected")+ }+ if rec.Code != http.StatusForbidden {+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusForbidden)+ }+}++func TestUploadMissingBucketAutoCreateDisabled(t *testing.T) {+ cases := []struct {+ name string+ method string+ object string+ handler func(*S3ApiServer, http.ResponseWriter, *http.Request)+ }{+ {"put object", http.MethodPut, "/key", (*S3ApiServer).PutObjectHandler},+ {"put directory marker", http.MethodPut, "/dir/", (*S3ApiServer).PutObjectHandler},+ {"new multipart upload", http.MethodPost, "/key", (*S3ApiServer).NewMultipartUploadHandler},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ s3a := &S3ApiServer{+ option: &S3ApiServerOption{},+ iam: &IdentityAccessManagement{},+ bucketConfigCache: NewBucketConfigCache(time.Minute),+ }+ s3a.bucketConfigCache.SetNegativeCache("missing")+ req := httptest.NewRequest(tc.method, "/missing"+tc.object, strings.NewReader(""))+ req = mux.SetURLVars(req, map[string]string{"bucket": "missing", "object": tc.object})+ rec := httptest.NewRecorder()++ tc.handler(s3a, rec, req)++ if rec.Code != http.StatusNotFound {+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)+ }+ if !strings.Contains(rec.Body.String(), "NoSuchBucket") {+ t.Fatalf("body = %s, want NoSuchBucket", rec.Body.String())+ }+ })+ }+}diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.goindex 95d6d9ed03..5f8652ac78 100644--- a/weed/s3api/s3api_object_handlers_put.go+++ b/weed/s3api/s3api_object_handlers_put.go@@ -131,6 +131,17 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)objectContentType := r.Header.Get("Content-Type")if strings.HasSuffix(object, "/") && r.ContentLength <= 1024 {+ // The filer mkdir below would implicitly create a missing bucket, so gate it+ // on the same auto-create policy as regular uploads.+ if err := s3a.checkBucket(r, bucket); err == s3err.ErrNoSuchBucket {+ if !s3a.handleAutoCreateBucket(w, r, bucket, "PutObjectHandler") {+ return+ }+ } else if err != s3err.ErrNone {+ s3err.WriteErrorResponse(w, r, err)+ return+ }+// Split the object into directory path and nameobjectWithoutSlash := strings.TrimSuffix(object, "/")dirName := path.Dir(objectWithoutSlash)diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.goindex 7b2e6c746f..7b09e08550 100644--- a/weed/s3api/s3api_server.go+++ b/weed/s3api/s3api_server.go@@ -52,6 +52,7 @@ type S3ApiServerOption struct {BucketsPath stringGrpcDialOption grpc.DialOptionAllowDeleteBucketNotEmpty bool+ AutoCreateBucket bool // create the bucket on upload if it does not existLocalFilerSocket stringDataCenter stringFilerGroup string