SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package shell
import (
"flag"
"fmt"
"io"
"path/filepath"
"regexp"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {
Commands = append(Commands, &commandRemoteMountBuckets{})
}
type commandRemoteMountBuckets struct {
}
func (c *commandRemoteMountBuckets) Name() string {
return "remote.mount.buckets"
}
func (c *commandRemoteMountBuckets) Help() string {
return `mount all buckets in remote storage and pull its metadata
# assume a remote storage is configured to name "cloud1"
remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
# mount all buckets
remote.mount.buckets -remote=cloud1
# after mount, start a separate process to write updates to remote storage
weed filer.remote.gateway -filer=<filerHost>:<filerPort> -createBucketAt=cloud1
`
}
func (c *commandRemoteMountBuckets) HasTag(CommandTag) bool {
return false
}
func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
remoteMountBucketsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
remote := remoteMountBucketsCommand.String("remote", "", "an already configured storage name")
bucketPattern := remoteMountBucketsCommand.String("bucketPattern", "", "match existing bucket name with wildcard characters '*' and '?'")
trimBucketSuffix := remoteMountBucketsCommand.Bool("trimBucketSuffix", true, "remote suffix auto generated by 'weed filer.remote.sync'")
apply := remoteMountBucketsCommand.Bool("apply", false, "apply the mount for listed buckets")
if err = remoteMountBucketsCommand.Parse(args); err != nil {
return err
}
infoAboutSimulationMode(writer, *apply, "-apply")
if *remote == "" {
_, err = listExistingRemoteStorageMounts(commandEnv, writer)
return err
}
// find configuration for remote storage
remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), *remote)
if err != nil {
return fmt.Errorf("find configuration for %s: %v", *remote, err)
}
// get storage client
remoteStorageClient, err := remote_storage.GetRemoteStorage(remoteConf)
if err != nil {
return fmt.Errorf("get storage client for %s: %v", *remote, err)
}
buckets, err := remoteStorageClient.ListBuckets()
if err != nil {
return fmt.Errorf("list buckets on %s: %v", *remote, err)
}
fillerBucketsPath, err := readFilerBucketsPath(commandEnv)
if err != nil {
return fmt.Errorf("read filer buckets path: %w", err)
}
hasSuffixPattern, _ := regexp.Compile(".+-[0-9][0-9][0-9][0-9]")
for _, bucket := range buckets {
if *bucketPattern != "" {
if matched, _ := filepath.Match(*bucketPattern, bucket.Name); !matched {
continue
}
}
fmt.Fprintf(writer, "bucket %s\n", bucket.Name)
localBucketName := bucket.Name
if *trimBucketSuffix {
if hasSuffixPattern.MatchString(localBucketName) {
localBucketName = localBucketName[:len(localBucketName)-5]
fmt.Fprintf(writer, " mount bucket %s as %s\n", bucket.Name, localBucketName)
}
}
if *apply {
dir := util.FullPath(fillerBucketsPath).Child(localBucketName)
remoteStorageLocation := &remote_pb.RemoteStorageLocation{
Name: *remote,
Bucket: bucket.Name,
Path: "/",
}
if err = ensureMountDirectory(commandEnv, string(dir), true, remoteConf); err != nil {
return fmt.Errorf("mount setup on %+v: %v", remoteStorageLocation, err)
}
if err = pullMetadata(commandEnv, writer, dir, remoteStorageLocation, dir, remoteConf, true, false); err != nil {
return fmt.Errorf("cache metadata on %+v: %v", remoteStorageLocation, err)
}
// store a mount configuration in filer
if err = filer.InsertMountMapping(commandEnv, string(dir), remoteStorageLocation); err != nil {
return fmt.Errorf("save mount mapping %s to %+v: %v", dir, remoteStorageLocation, err)
}
saveMountSyncOffset(commandEnv, writer, string(dir))
}
}
return nil
}