SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package weed_server
import (
"context"
"fmt"
"math/rand/v2"
"sync"
"time"
"github.com/seaweedfs/raft"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
/*
How exclusive lock works?
-----------
Shell
------
When shell lock,
* lease an admin token (lockTime, token)
* start a goroutine to renew the admin token periodically
When shell unlock
* stop the renewal goroutine
* sends a release lock request
Master
------
Master maintains:
* randomNumber
* lastLockTime
When master receives the lease/renew request from shell
If lastLockTime still fresh {
if is a renew and token is valid {
// for renew
generate the randomNumber => token
return
}
refuse
return
} else {
// for fresh lease request
generate the randomNumber => token
return
}
When master receives the release lock request from shell
set the lastLockTime to zero
The volume server does not need to verify.
This makes the lock/unlock optional, similar to what golang code usually does.
*/
const (
LockDuration = 10 * time.Second
)
type AdminLock struct {
accessSecret int64
accessLockTime time.Time
lastClient string
lastMessage string
}
type AdminLocks struct {
locks map[string]*AdminLock
sync.RWMutex
}
func NewAdminLocks() *AdminLocks {
return &AdminLocks{
locks: make(map[string]*AdminLock),
}
}
func (locks *AdminLocks) isLocked(lockName string) (clientName string, message string, isLocked bool) {
locks.RLock()
defer locks.RUnlock()
return locks.lockStatusLocked(lockName)
}
// The *Locked helpers below expect the caller to hold locks, so that a lease
// or release checks and changes the lock state in one critical section.
func (locks *AdminLocks) lockStatusLocked(lockName string) (clientName string, message string, isLocked bool) {
adminLock, found := locks.locks[lockName]
if !found {
return "", "", false
}
glog.V(4).Infof("isLocked %v: %v", adminLock.lastClient, adminLock.lastMessage)
return adminLock.lastClient, adminLock.lastMessage, adminLock.accessLockTime.Add(LockDuration).After(time.Now())
}
func (locks *AdminLocks) isValidTokenLocked(lockName string, ts time.Time, token int64) bool {
adminLock, found := locks.locks[lockName]
if !found {
return false
}
return adminLock.accessLockTime.Equal(ts) && adminLock.accessSecret == token
}
func (locks *AdminLocks) generateTokenLocked(lockName string, clientName string) (ts time.Time, token int64) {
if existing, found := locks.locks[lockName]; found && existing.lastClient != clientName {
stats.MasterAdminLock.DeleteLabelValues(existing.lastClient)
}
lock := &AdminLock{
accessSecret: rand.Int64(),
accessLockTime: time.Now(),
lastClient: clientName,
}
locks.locks[lockName] = lock
stats.MasterAdminLock.WithLabelValues(clientName).Set(1)
return lock.accessLockTime, lock.accessSecret
}
func (locks *AdminLocks) deleteLockLocked(lockName string) {
if adminLock, found := locks.locks[lockName]; found {
stats.MasterAdminLock.DeleteLabelValues(adminLock.lastClient)
delete(locks.locks, lockName)
}
}
func (ms *MasterServer) LeaseAdminToken(ctx context.Context, req *master_pb.LeaseAdminTokenRequest) (*master_pb.LeaseAdminTokenResponse, error) {
resp := &master_pb.LeaseAdminTokenResponse{}
if !ms.Topo.IsLeader() {
return resp, raft.NotLeaderError
}
ms.adminLocks.Lock()
defer ms.adminLocks.Unlock()
if lastClient, lastMessage, isLocked := ms.adminLocks.lockStatusLocked(req.LockName); isLocked {
glog.V(4).Infof("LeaseAdminToken %v", lastClient)
if req.PreviousToken != 0 && ms.adminLocks.isValidTokenLocked(req.LockName, time.Unix(0, req.PreviousLockTime), req.PreviousToken) {
// for renew
ts, token := ms.adminLocks.generateTokenLocked(req.LockName, req.ClientName)
resp.Token, resp.LockTsNs = token, ts.UnixNano()
return resp, nil
}
// refuse since still locked
return resp, fmt.Errorf("already locked by %v: %v", lastClient, lastMessage)
}
// for fresh lease request
ts, token := ms.adminLocks.generateTokenLocked(req.LockName, req.ClientName)
resp.Token, resp.LockTsNs = token, ts.UnixNano()
return resp, nil
}
func (ms *MasterServer) ReleaseAdminToken(ctx context.Context, req *master_pb.ReleaseAdminTokenRequest) (*master_pb.ReleaseAdminTokenResponse, error) {
resp := &master_pb.ReleaseAdminTokenResponse{}
// a follower holds no lock state; answering success here would fake a
// release while the leader keeps the lock until it expires
if !ms.Topo.IsLeader() {
return resp, raft.NotLeaderError
}
ms.adminLocks.Lock()
defer ms.adminLocks.Unlock()
if ms.adminLocks.isValidTokenLocked(req.LockName, time.Unix(0, req.PreviousLockTime), req.PreviousToken) {
ms.adminLocks.deleteLockLocked(req.LockName)
}
return resp, nil
}
func (ms *MasterServer) GetAdminLockStatus(ctx context.Context, req *master_pb.GetAdminLockStatusRequest) (*master_pb.GetAdminLockStatusResponse, error) {
resp := &master_pb.GetAdminLockStatusResponse{}
if !ms.Topo.IsLeader() {
return resp, raft.NotLeaderError
}
// isLocked reports the last holder even after the lease expired
if clientName, message, isLocked := ms.adminLocks.isLocked(req.LockName); isLocked {
resp.IsLocked = true
resp.ClientName = clientName
resp.Message = message
}
return resp, nil
}
// isKnownPingTarget reports whether target is a peer that the master has
// learned about as part of cluster membership. Restricting Ping to known
// peers avoids turning the RPC into a generic outbound dialer. The lookups
// are O(1) so the gate stays cheap on clusters with thousands of nodes.
func (ms *MasterServer) isKnownPingTarget(ctx context.Context, target string, targetType string) bool {
addr := pb.ServerAddress(target)
switch targetType {
case cluster.FilerType, cluster.BrokerType, cluster.S3Type:
return ms.Cluster.IsKnownNode(targetType, addr)
case cluster.VolumeServerType:
if ms.Topo != nil && ms.Topo.LookupDataNodeByAddress(addr) != nil {
return true
}
// Only the leader receives volume-server heartbeats, so a follower's
// topology is empty. Fall back to the volume-server set the master
// learns over its own MasterClient subscription to the leader, the
// same source the filer gate trusts.
if ms.MasterClient != nil {
return ms.MasterClient.HasVolumeServer(addr)
}
return false
case cluster.MasterType:
if ms.option != nil && ms.option.Master.Equals(addr) {
return true
}
if ms.MasterClient != nil {
_, ok := ms.MasterClient.ListMasterSet()[addr.ToHttpAddress()]
return ok
}
return false
}
return false
}
func (ms *MasterServer) Ping(ctx context.Context, req *master_pb.PingRequest) (resp *master_pb.PingResponse, pingErr error) {
resp = &master_pb.PingResponse{
StartTimeNs: time.Now().UnixNano(),
}
// Empty target is a self-liveness probe and stays unauthenticated.
if req.Target != "" && !ms.isKnownPingTarget(ctx, req.Target, req.TargetType) {
resp.StopTimeNs = time.Now().UnixNano()
return resp, status.Errorf(codes.InvalidArgument, "unknown ping target %s of type %s", req.Target, req.TargetType)
}
if req.TargetType == cluster.FilerType {
pingErr = pb.WithFilerClient(false, 0, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if req.TargetType == cluster.VolumeServerType {
pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if req.TargetType == cluster.MasterType {
pingErr = pb.WithMasterClient(context.Background(), false, pb.ServerAddress(req.Target), ms.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
if pingResp != nil {
resp.RemoteTimeNs = pingResp.StartTimeNs
}
return err
})
}
if pingErr != nil {
pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr)
}
resp.StopTimeNs = time.Now().UnixNano()
return
}