Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

4.0 KB
Raw Blame
package shell

import (
	"flag"
	"fmt"
	"io"
	"path/filepath"
	"regexp"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandRemoteMountBuckets{})
}

type commandRemoteMountBuckets struct {
}

func (c *commandRemoteMountBuckets) Name() string {
	return "remote.mount.buckets"
}

func (c *commandRemoteMountBuckets) Help() string {
	return `mount all buckets in remote storage and pull its metadata

	# assume a remote storage is configured to name "cloud1"
	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy

	# mount all buckets
	remote.mount.buckets -remote=cloud1

	# after mount, start a separate process to write updates to remote storage
	weed filer.remote.gateway -filer=<filerHost>:<filerPort> -createBucketAt=cloud1

`
}

func (c *commandRemoteMountBuckets) HasTag(CommandTag) bool {
	return false
}

func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteMountBucketsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	remote := remoteMountBucketsCommand.String("remote", "", "an already configured storage name")
	bucketPattern := remoteMountBucketsCommand.String("bucketPattern", "", "match existing bucket name with wildcard characters '*' and '?'")
	trimBucketSuffix := remoteMountBucketsCommand.Bool("trimBucketSuffix", true, "remote suffix auto generated by 'weed filer.remote.sync'")
	apply := remoteMountBucketsCommand.Bool("apply", false, "apply the mount for listed buckets")
	if err = remoteMountBucketsCommand.Parse(args); err != nil {
		return err
	}
	infoAboutSimulationMode(writer, *apply, "-apply")

	if *remote == "" {
		_, err = listExistingRemoteStorageMounts(commandEnv, writer)
		return err
	}

	// find configuration for remote storage
	remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), *remote)
	if err != nil {
		return fmt.Errorf("find configuration for %s: %v", *remote, err)
	}

	// get storage client
	remoteStorageClient, err := remote_storage.GetRemoteStorage(remoteConf)
	if err != nil {
		return fmt.Errorf("get storage client for %s: %v", *remote, err)
	}

	buckets, err := remoteStorageClient.ListBuckets()
	if err != nil {
		return fmt.Errorf("list buckets on %s: %v", *remote, err)
	}

	fillerBucketsPath, err := readFilerBucketsPath(commandEnv)
	if err != nil {
		return fmt.Errorf("read filer buckets path: %w", err)
	}

	hasSuffixPattern, _ := regexp.Compile(".+-[0-9][0-9][0-9][0-9]")

	for _, bucket := range buckets {
		if *bucketPattern != "" {
			if matched, _ := filepath.Match(*bucketPattern, bucket.Name); !matched {
				continue
			}
		}

		fmt.Fprintf(writer, "bucket %s\n", bucket.Name)
		localBucketName := bucket.Name
		if *trimBucketSuffix {
			if hasSuffixPattern.MatchString(localBucketName) {
				localBucketName = localBucketName[:len(localBucketName)-5]
				fmt.Fprintf(writer, "  mount bucket %s as %s\n", bucket.Name, localBucketName)
			}
		}
		if *apply {

			dir := util.FullPath(fillerBucketsPath).Child(localBucketName)
			remoteStorageLocation := &remote_pb.RemoteStorageLocation{
				Name:   *remote,
				Bucket: bucket.Name,
				Path:   "/",
			}

			if err = ensureMountDirectory(commandEnv, string(dir), true, remoteConf); err != nil {
				return fmt.Errorf("mount setup on %+v: %v", remoteStorageLocation, err)
			}
			if err = pullMetadata(commandEnv, writer, dir, remoteStorageLocation, dir, remoteConf, true, false); err != nil {
				return fmt.Errorf("cache metadata on %+v: %v", remoteStorageLocation, err)
			}

			// store a mount configuration in filer
			if err = filer.InsertMountMapping(commandEnv, string(dir), remoteStorageLocation); err != nil {
				return fmt.Errorf("save mount mapping %s to %+v: %v", dir, remoteStorageLocation, err)
			}
			saveMountSyncOffset(commandEnv, writer, string(dir))

		}
	}

	return nil
}