Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

feat(mini): add -bucket flag to pre-create an S3 bucket on startup (#9302)

Chris Luf16353de0bf8
diff --git a/README.md b/README.md
index 2a41423b04..aa415704d0 100644
--- a/README.md
+++ b/README.md
@@ -80,28 +80,29 @@ Table of Contents
## Quick Start with weed mini ##
-The easiest way to get started with SeaweedFS for development and testing:
-* Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip a single binary file `weed` or `weed.exe`.
-
-Example:
+Download the latest binary from https://github.com/seaweedfs/seaweedfs/releases and unzip the single `weed` (or `weed.exe`) file. Then start a ready-to-use S3 object store with credentials and a pre-created bucket in one command:
```bash
-# remove quarantine on macOS
-# xattr -d com.apple.quarantine ./weed
-
+AWS_ACCESS_KEY_ID=admin \
+AWS_SECRET_ACCESS_KEY=secret \
+S3_BUCKET=my-bucket \
./weed mini -dir=/data
```
-This single command starts a complete SeaweedFS setup with:
+That's it — the S3 endpoint is at http://localhost:8333, `my-bucket` already exists, and `admin`/`secret` are valid credentials. `S3_BUCKET` accepts a comma-separated list (e.g. `raw,processed`); use `S3_TABLE_BUCKET` for S3 Tables (Iceberg) buckets. Drop any of the env vars to skip that piece (no AWS keys → S3 runs in unauthenticated "Allow All" mode for development).
+
+The same command starts everything else too:
+- **S3 Endpoint**: http://localhost:8333
- **Master UI**: http://localhost:9333
- **Volume Server**: http://localhost:9340
- **Filer UI**: http://localhost:8888
-- **S3 Endpoint**: http://localhost:8333
- **WebDAV**: http://localhost:7333
- **Admin UI**: http://localhost:23646
-Perfect for development, testing, learning SeaweedFS, and single node deployments!
+> macOS: if the binary is quarantined, run `xattr -d com.apple.quarantine ./weed` first.
+
+Perfect for development, testing, learning SeaweedFS, and single-node deployments.
## Quick Start for S3 API on Docker ##
diff --git a/weed/command/mini.go b/weed/command/mini.go
index 949b1ce73d..10c57c444f 100644
--- a/weed/command/mini.go
+++ b/weed/command/mini.go
@@ -2,6 +2,7 @@ package command
import (
"context"
+ "errors"
"fmt"
"math/bits"
"net"
@@ -14,7 +15,10 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
+ "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
pluginworker "github.com/seaweedfs/seaweedfs/weed/plugin/worker"
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket"
+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables"
"github.com/seaweedfs/seaweedfs/weed/security"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
@@ -212,6 +216,9 @@ Example Usage:
weed mini # Use current directory
weed mini -dir=/data # Custom data directory
weed mini -dir=/data -master.port=9444 # Custom master port
+ weed mini -dir=/data -bucket=my-bucket # Pre-create an S3 bucket on startup
+ weed mini -dir=/data -bucket=bucket1,bucket2 # Pre-create multiple S3 buckets
+ weed mini -dir=/data -tableBucket=iceberg-tables # Pre-create an S3 Tables bucket
After starting, you can access:
- Master UI: http://localhost:9333
@@ -244,6 +251,8 @@ var (
miniS3Config = cmdMini.Flag.String("s3.config", "", "path to the S3 config file")
miniIamConfig = cmdMini.Flag.String("s3.iam.config", "", "path to the advanced IAM config file for S3")
miniS3AllowDeleteBucketNotEmpty = cmdMini.Flag.Bool("s3.allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
+ miniBucket = cmdMini.Flag.String("bucket", "", "comma-separated S3 bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_BUCKET env var.")
+ miniTableBucket = cmdMini.Flag.String("tableBucket", "", "comma-separated S3 Tables bucket names to create on startup if they do not already exist; leave empty to skip. Falls back to S3_TABLE_BUCKET env var.")
)
// getBindIp determines the bind IP address based on miniIp and miniBindIp flags
@@ -1020,6 +1029,22 @@ func runMini(cmd *Command, args []string) bool {
triggerMiniClientsShutdown(10 * time.Second)
})
+ // Create the requested bucket(s) (if any) before announcing readiness.
+ bucketSpec := *miniBucket
+ if bucketSpec == "" {
+ bucketSpec = os.Getenv("S3_BUCKET")
+ }
+ if err := ensureMiniBuckets(bucketSpec); err != nil {
+ glog.Warningf("failed to ensure buckets %q: %v", bucketSpec, err)
+ }
+ tableBucketSpec := *miniTableBucket
+ if tableBucketSpec == "" {
+ tableBucketSpec = os.Getenv("S3_TABLE_BUCKET")
+ }
+ if err := ensureMiniTableBuckets(tableBucketSpec); err != nil {
+ glog.Warningf("failed to ensure table buckets %q: %v", tableBucketSpec, err)
+ }
+
// Print welcome message after all services are running
printWelcomeMessage()
@@ -1484,3 +1509,111 @@ func printWelcomeMessage() {
fmt.Print(sb.String())
fmt.Println("")
}
+
+// ensureMiniBuckets creates each named bucket on the embedded filer if it does
+// not already exist. bucketSpec is a comma-separated list (whitespace around
+// each name is trimmed); empty entries and an empty spec are no-ops so callers
+// who do not pass -bucket pay nothing. Per-bucket failures are logged and the
+// loop continues so a single bad name does not block creating the rest.
+func ensureMiniBuckets(bucketSpec string) error {
+ names := parseBucketList(bucketSpec)
+ if len(names) == 0 {
+ return nil
+ }
+
+ filerAddress := pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc)
+ grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+ const bucketsPath = "/buckets"
+ // Derive from miniClientsCtx so Ctrl+C cancels the bucket RPCs, and bound
+ // with a short timeout (per bucket) so a stalled filer cannot block the
+ // welcome message indefinitely.
+ return pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+ for _, name := range names {
+ if err := s3bucket.VerifyS3BucketName(name); err != nil {
+ glog.Warningf("invalid bucket name %q: %v", name, err)
+ continue
+ }
+ ctx, cancel := context.WithTimeout(miniClientsCtx(), 5*time.Second)
+ _, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{
+ Directory: bucketsPath,
+ Name: name,
+ })
+ if err == nil {
+ glog.V(0).Infof("bucket %s already exists", name)
+ cancel()
+ continue
+ }
+ if !errors.Is(err, filer_pb.ErrNotFound) {
+ glog.Warningf("lookup bucket %s: %v", name, err)
+ cancel()
+ continue
+ }
+ if err := filer_pb.DoMkdir(ctx, client, bucketsPath, name, nil); err != nil {
+ glog.Warningf("create bucket %s: %v", name, err)
+ cancel()
+ continue
+ }
+ cancel()
+ glog.V(0).Infof("created bucket %s", name)
+ }
+ return nil
+ })
+}
+
+// ensureMiniTableBuckets creates each named S3 Tables bucket on the embedded
+// filer if it does not already exist. bucketSpec is comma-separated; whitespace
+// is trimmed and duplicates are dropped. Per-bucket failures are logged so one
+// bad name does not block the rest. Buckets are owned by s3tables.DefaultAccountID
+// since mini does not yet model multi-account ownership.
+func ensureMiniTableBuckets(bucketSpec string) error {
+ names := parseBucketList(bucketSpec)
+ if len(names) == 0 {
+ return nil
+ }
+
+ filerAddress := pb.NewServerAddress(*miniIp, *miniFilerOptions.port, *miniFilerOptions.portGrpc)
+ grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+ return pb.WithGrpcFilerClient(false, 0, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+ manager := s3tables.NewManager()
+ mgrClient := s3tables.NewManagerClient(client)
+ for _, name := range names {
+ ctx, cancel := context.WithTimeout(miniClientsCtx(), 5*time.Second)
+ req := &s3tables.CreateTableBucketRequest{Name: name}
+ var resp s3tables.CreateTableBucketResponse
+ err := manager.Execute(ctx, mgrClient, "CreateTableBucket", req, &resp, s3tables.DefaultAccountID)
+ cancel()
+ if err == nil {
+ glog.V(0).Infof("created table bucket %s", name)
+ continue
+ }
+ var s3Err *s3tables.S3TablesError
+ if errors.As(err, &s3Err) && s3Err.Type == s3tables.ErrCodeBucketAlreadyExists {
+ glog.V(0).Infof("table bucket %s already exists", name)
+ continue
+ }
+ glog.Warningf("create table bucket %s: %v", name, err)
+ }
+ return nil
+ })
+}
+
+// parseBucketList splits a comma-separated bucket spec into a deduplicated list
+// of trimmed, non-empty names, preserving the order they were given.
+func parseBucketList(spec string) []string {
+ if spec == "" {
+ return nil
+ }
+ seen := make(map[string]bool)
+ var names []string
+ for _, raw := range strings.Split(spec, ",") {
+ name := strings.TrimSpace(raw)
+ if name == "" || seen[name] {
+ continue
+ }
+ seen[name] = true
+ names = append(names, name)
+ }
+ return names
+}
diff --git a/weed/command/mini_bucket_test.go b/weed/command/mini_bucket_test.go
new file mode 100644
index 0000000000..4a2ac66db4
--- /dev/null
+++ b/weed/command/mini_bucket_test.go
@@ -0,0 +1,30 @@
+package command
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestParseBucketList(t *testing.T) {
+ tests := []struct {
+ name string
+ in string
+ want []string
+ }{
+ {"empty", "", nil},
+ {"single", "one", []string{"one"}},
+ {"multi", "one,two,three", []string{"one", "two", "three"}},
+ {"trims whitespace", " one , two , three ", []string{"one", "two", "three"}},
+ {"drops empty entries", "one,,two,", []string{"one", "two"}},
+ {"dedupes preserving order", "one,two,one,three,two", []string{"one", "two", "three"}},
+ {"only commas", ",,,", nil},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := parseBucketList(tt.in)
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("parseBucketList(%q) = %v, want %v", tt.in, got, tt.want)
+ }
+ })
+ }
+}