SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
//go:build linux || darwin || freebsd || windows
package command
import (
"context"
"errors"
"fmt"
"math"
"net"
"net/http"
"os"
"path"
"runtime"
"runtime/debug"
"strconv"
"strings"
"sync"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/reflection"
"google.golang.org/grpc/status"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mount"
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mount_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
)
func runMount(cmd *Command, args []string) bool {
if *mountOptions.debug {
go http.ListenAndServe(fmt.Sprintf(":%d", *mountOptions.debugPort), nil)
}
*mountCpuProfile = util.ResolvePath(*mountCpuProfile)
*mountMemProfile = util.ResolvePath(*mountMemProfile)
grace.SetupProfiling(*mountCpuProfile, *mountMemProfile)
if *mountReadRetryTime < time.Second {
*mountReadRetryTime = time.Second
}
util.RetryWaitTime = *mountReadRetryTime
// 32 bits, not 64: os.FileMode is uint32, so a wider parse would let a
// nonsense umask truncate silently instead of being rejected here.
umask, umaskErr := strconv.ParseUint(*mountOptions.umaskString, 8, 32)
if umaskErr != nil {
fmt.Printf("can not parse umask %s", *mountOptions.umaskString)
return false
}
if len(args) > 0 {
return false
}
return RunMount(&mountOptions, os.FileMode(umask))
}
func ensureBucketAllowEmptyFolders(ctx context.Context, filerClient filer_pb.FilerClient, mountRoot, bucketRootPath string) error {
bucketPath, isBucketRootMount := bucketPathForMountRoot(mountRoot, bucketRootPath)
if !isBucketRootMount {
return nil
}
bucketFullPath := util.FullPath(bucketPath)
parent, _ := bucketFullPath.DirAndName()
for attempt := 0; attempt < 5; attempt++ {
entry, _, _, err := filer_pb.GetEntry(ctx, filerClient, util.FullPath(bucketPath))
if err != nil {
return err
}
if entry == nil {
return fmt.Errorf("bucket %s not found", bucketPath)
}
if value := strings.TrimSpace(string(entry.Extended[s3_constants.ExtAllowEmptyFolders])); value != "" {
return nil
}
expected := filer_pb.SnapshotExtended(entry.Extended, s3_constants.ExtAllowEmptyFolders)
if entry.Extended == nil {
entry.Extended = make(map[string][]byte)
}
entry.Extended[s3_constants.ExtAllowEmptyFolders] = []byte("true")
err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
_, err := client.UpdateEntry(ctx, &filer_pb.UpdateEntryRequest{
Directory: parent,
Entry: entry,
ExpectedExtended: expected,
})
return err
})
if err == nil {
glog.V(3).Infof("RunMount: set bucket %s %s=true", bucketPath, s3_constants.ExtAllowEmptyFolders)
return nil
}
if status.Code(err) != codes.FailedPrecondition {
return err
}
}
return fmt.Errorf("bucket %s attributes keep changing", bucketPath)
}
func bucketPathForMountRoot(mountRoot, bucketRootPath string) (string, bool) {
cleanPath := path.Clean("/" + strings.TrimPrefix(mountRoot, "/"))
cleanBucketRoot := path.Clean("/" + strings.TrimPrefix(bucketRootPath, "/"))
if cleanBucketRoot == "/" {
return "", false
}
prefix := cleanBucketRoot + "/"
if !strings.HasPrefix(cleanPath, prefix) {
return "", false
}
rest := strings.TrimPrefix(cleanPath, prefix)
bucketParts := strings.Split(rest, "/")
if len(bucketParts) != 1 || bucketParts[0] == "" {
return "", false
}
return cleanBucketRoot + "/" + bucketParts[0], true
}
func peerStringOrEmpty(p *string) string {
if p == nil {
return ""
}
return *p
}
// connectToFiler retries the filer handshake, returning the cluster's cipher
// setting and bucket root.
func connectToFiler(option *MountOptions) (filerAddresses []pb.ServerAddress, grpcDialOption grpc.DialOption, cipher bool, bucketRootPath string, ok bool) {
// try to connect to filer
filerAddresses = pb.ServerAddresses(*option.filer).ToAddresses()
util.LoadSecurityConfiguration()
grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
var err error
for i := 0; i < 10; i++ {
err = pb.WithOneOfGrpcFilerClients(false, filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return fmt.Errorf("get filer grpc address %v configuration: %w", filerAddresses, err)
}
cipher = resp.Cipher
bucketRootPath = resp.DirBuckets
return nil
})
if err == nil {
break
}
glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err)
glog.V(0).Infof("wait for %d seconds ...", i+1)
time.Sleep(time.Duration(i+1) * time.Second)
}
if err != nil {
glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
return nil, nil, false, "", false
}
if bucketRootPath == "" {
bucketRootPath = "/buckets"
}
return filerAddresses, grpcDialOption, cipher, bucketRootPath, true
}
// fileSystemParams are the pieces of a mount that each platform works out for
// itself: where it is attached, and whose identity the entries carry.
type fileSystemParams struct {
dir string
mountRoot string
filerAddresses []pb.ServerAddress
grpcDialOption grpc.DialOption
cipher bool
uidGidMapper *meta_cache.UidGidMapper
uid uint32
gid uint32
mountMode os.FileMode
mountCtime time.Time
umask os.FileMode
chunkSizeLimitMB int
cacheDirForRead string
cacheDirForWrite string
// eagerFilerCreate persists a created file's entry at create time rather
// than at flush. A platform sets it when it cannot run the flush before
// the application's close returns, so nothing that reads through the
// filer can race an unflushed close.
eagerFilerCreate bool
}
func buildSeaweedFileSystem(option *MountOptions, p fileSystemParams) *mount.WFS {
readerCacheSizeMB := int64(256)
if option.readerCacheSizeMB != nil {
readerCacheSizeMB = *option.readerCacheSizeMB
}
return mount.NewSeaweedFileSystem(&mount.Option{
MountDirectory: p.dir,
FilerAddresses: p.filerAddresses,
GrpcDialOption: p.grpcDialOption,
FilerSigningKey: security.SigningKey(util.GetViper().GetString("jwt.filer_signing.key")),
FilerSigningExpiresAfterSec: util.GetViper().GetInt("jwt.filer_signing.expires_after_seconds"),
FilerMountRootPath: p.mountRoot,
Collection: *option.collection,
Replication: *option.replication,
TtlSec: int32(*option.ttlSec),
DiskType: types.ToDiskType(*option.diskType),
ChunkSizeLimit: int64(p.chunkSizeLimitMB) * 1024 * 1024,
ConcurrentWriters: *option.concurrentWriters,
ConcurrentReaders: *option.concurrentReaders,
ReaderCacheSizeMB: readerCacheSizeMB,
CacheDirForRead: p.cacheDirForRead,
CacheSizeMBForRead: *option.cacheSizeMBForRead,
CacheDirForWrite: p.cacheDirForWrite,
WriteBufferSizeMB: *option.writeBufferSizeMB,
CacheMetaTTlSec: *option.cacheMetaTtlSec,
CacheDirMaxEntries: *option.cacheDirMaxEntries,
DataCenter: *option.dataCenter,
Quota: int64(*option.collectionQuota) * 1024 * 1024,
LogicalDiskUsage: *option.logicalDiskUsage,
MountUid: p.uid,
MountGid: p.gid,
MountMode: p.mountMode,
MountCtime: p.mountCtime,
MountMtime: time.Now(),
Umask: p.umask,
VolumeServerAccess: *mountOptions.volumeServerAccess,
Cipher: p.cipher,
UidGidMapper: p.uidGidMapper,
IncludeSystemEntries: *option.includeSystemEntries,
DefaultPermissions: *option.defaultPermissions,
DisableXAttr: *option.disableXAttr,
IsMacOs: runtime.GOOS == "darwin",
MetadataFlushSeconds: *option.metadataFlushSeconds,
DirIdleEvictSec: *option.dirIdleEvictSec,
EnableDistributedLock: option.distributedLock != nil && *option.distributedLock,
WritebackCache: option.writebackCache != nil && *option.writebackCache,
EagerFilerCreate: p.eagerFilerCreate,
PosixDirNlink: option.posixDirNlink != nil && *option.posixDirNlink,
// Peer chunk sharing
PeerEnabled: option.peerEnabled != nil && *option.peerEnabled,
PeerListen: peerStringOrEmpty(option.peerListen),
PeerAdvertise: peerStringOrEmpty(option.peerAdvertise),
PeerDataCenter: peerStringOrEmpty(option.peerDataCenter),
PeerRack: peerStringOrEmpty(option.peerRack),
})
}
// createMountRoot makes the filer-side directory the mount is rooted at.
func createMountRoot(wfs *mount.WFS, mountRoot, bucketRootPath string, filerAddresses []pb.ServerAddress) bool {
if err := ensureMountRootDirectory(context.Background(), wfs, mountRoot); err != nil {
fmt.Printf("failed to create dir %s on filer %s: %v\n", mountRoot, filerAddresses, err)
return false
}
if err := ensureBucketAllowEmptyFolders(context.Background(), wfs, mountRoot, bucketRootPath); err != nil {
fmt.Printf("failed to set bucket auto-remove-empty-folders policy for %s: %v\n", mountRoot, err)
return false
}
return true
}
// ensureMountRootDirectory creates the mount root only when it is missing. The
// filer answers a non-exclusive create of an existing path by replacing the
// entry, keeping only its crtime and inode, so creating the root again at every
// mount start would reset its mode and owner and drop what only the filer
// holds: its extended attributes and, when the root is a bucket, the bucket's
// quota and the settings S3 keeps on it.
func ensureMountRootDirectory(ctx context.Context, filerClient filer_pb.FilerClient, mountRoot string) error {
mountRootPath := util.FullPath(mountRoot)
if mountRootPath == "/" {
return nil
}
entry, _, _, err := filer_pb.GetEntry(ctx, filerClient, mountRootPath)
if err == nil {
return mountRootMustBeDirectory(mountRoot, entry)
}
if !errors.Is(err, filer_pb.ErrNotFound) {
return err
}
parent, name := mountRootPath.DirAndName()
if err := filer_pb.MkdirExclusive(ctx, filerClient, parent, name, nil); err != nil {
// Another client may have created it since the lookup. A filer that
// predates error codes reports that only as text, so look again rather
// than match the error.
entry, _, _, lookupErr := filer_pb.GetEntry(ctx, filerClient, mountRootPath)
if lookupErr != nil {
return err
}
return mountRootMustBeDirectory(mountRoot, entry)
}
return nil
}
func mountRootMustBeDirectory(mountRoot string, entry *filer_pb.Entry) error {
if !entry.GetIsDirectory() {
return fmt.Errorf("%s: %w", mountRoot, filer_pb.ErrExistingIsFile)
}
return nil
}
// mountSocketCleanup returns a func that removes the control socket this mount
// listens on, unless a later mount of the same directory has replaced it.
func mountSocketCleanup(socketPath string) func() {
listened, err := os.Lstat(socketPath)
if err != nil {
return func() {}
}
var once sync.Once
return func() {
once.Do(func() {
current, err := os.Lstat(socketPath)
if err != nil || !os.SameFile(current, listened) {
return
}
if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
glog.Warningf("remove mount socket %s: %v", socketPath, err)
}
})
}
}
// serveMountGrpc exposes the local control socket used by "weed mount.stats".
func serveMountGrpc(wfs *mount.WFS, listener net.Listener) {
grpcS := pb.NewGrpcServer()
mount_pb.RegisterSeaweedMountServer(grpcS, wfs)
reflection.Register(grpcS)
go grpcS.Serve(listener)
}
// resolveMountRoot trims the trailing slash the filer path must not carry.
func resolveMountRoot(filerMountRootPath string) string {
mountRoot := filerMountRootPath
if mountRoot != "/" && strings.HasSuffix(mountRoot, "/") {
mountRoot = mountRoot[0 : len(mountRoot)-1]
}
return mountRoot
}
// resolveCacheDirs falls back to the read cache when no write cache is set.
func resolveCacheDirs(option *MountOptions) (string, string) {
cacheDirForRead := util.ResolvePath(*option.cacheDirForRead)
cacheDirForWrite := util.ResolvePath(*option.cacheDirForWrite)
if cacheDirForWrite == "" {
cacheDirForWrite = cacheDirForRead
}
return cacheDirForRead, cacheDirForWrite
}
// volumeName labels the mount where the platform shows one, in Finder and in
// Explorer. An explicit override always wins; failing that, the mounted path
// names the disk, then the mount point; the filer address, which every mount
// from one filer shares, is the last resort.
func volumeName(filer, filerMountRootPath, dir, override string) string {
name := override
if name == "" {
name = lastSegment(filerMountRootPath)
}
if name == "" {
name = lastSegment(dir)
}
if name == "" {
name = filer
}
return strings.ReplaceAll(name, ",", "+")
}
// lastSegment is the name a path ends with, and "" for the ones that name no
// disk: the root, "." and a bare Windows drive letter.
func lastSegment(p string) string {
name := path.Base(strings.ReplaceAll(p, `\`, "/"))
if name == "/" || name == "." || strings.HasSuffix(name, ":") {
return ""
}
return name
}
func configureMountMemory(option *MountOptions) error {
if option.readerCacheSizeMB != nil && (*option.readerCacheSizeMB <= 0 || *option.readerCacheSizeMB > math.MaxInt64>>20) {
return fmt.Errorf("readerCacheSizeMB must be positive and fit in an int64 byte budget")
}
if option.memoryLimitMB != nil {
if *option.memoryLimitMB < 0 || *option.memoryLimitMB > math.MaxInt64>>20 {
return fmt.Errorf("memoryLimitMB must be non-negative and fit in an int64 byte limit")
}
if *option.memoryLimitMB > 0 {
debug.SetMemoryLimit(*option.memoryLimitMB << 20)
}
}
return nil
}