Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

8.5 KB
Raw Blame
package weed_server

import (
	"context"
	"fmt"
	"math/rand/v2"
	"sync"
	"time"

	"github.com/seaweedfs/raft"
	"github.com/seaweedfs/seaweedfs/weed/cluster"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

/*
How exclusive lock works?
-----------

Shell
------
When shell lock,
  * lease an admin token (lockTime, token)
  * start a goroutine to renew the admin token periodically

When shell unlock
  * stop the renewal goroutine
  * sends a release lock request

Master
------
Master maintains:
  * randomNumber
  * lastLockTime
When master receives the lease/renew request from shell
  If lastLockTime still fresh {
    if is a renew and token is valid {
      // for renew
      generate the randomNumber => token
      return
    }
    refuse
    return
  } else {
    // for fresh lease request
    generate the randomNumber => token
    return
  }

When master receives the release lock request from shell
  set the lastLockTime to zero


The volume server does not need to verify.
This makes the lock/unlock optional, similar to what golang code usually does.

*/

const (
	LockDuration = 10 * time.Second
)

type AdminLock struct {
	accessSecret   int64
	accessLockTime time.Time
	lastClient     string
	lastMessage    string
}

type AdminLocks struct {
	locks map[string]*AdminLock
	sync.RWMutex
}

func NewAdminLocks() *AdminLocks {
	return &AdminLocks{
		locks: make(map[string]*AdminLock),
	}
}

func (locks *AdminLocks) isLocked(lockName string) (clientName string, message string, isLocked bool) {
	locks.RLock()
	defer locks.RUnlock()
	return locks.lockStatusLocked(lockName)
}

// The *Locked helpers below expect the caller to hold locks, so that a lease
// or release checks and changes the lock state in one critical section.
func (locks *AdminLocks) lockStatusLocked(lockName string) (clientName string, message string, isLocked bool) {
	adminLock, found := locks.locks[lockName]
	if !found {
		return "", "", false
	}
	glog.V(4).Infof("isLocked %v: %v", adminLock.lastClient, adminLock.lastMessage)
	return adminLock.lastClient, adminLock.lastMessage, adminLock.accessLockTime.Add(LockDuration).After(time.Now())
}

func (locks *AdminLocks) isValidTokenLocked(lockName string, ts time.Time, token int64) bool {
	adminLock, found := locks.locks[lockName]
	if !found {
		return false
	}
	return adminLock.accessLockTime.Equal(ts) && adminLock.accessSecret == token
}

func (locks *AdminLocks) generateTokenLocked(lockName string, clientName string) (ts time.Time, token int64) {
	if existing, found := locks.locks[lockName]; found && existing.lastClient != clientName {
		stats.MasterAdminLock.DeleteLabelValues(existing.lastClient)
	}
	lock := &AdminLock{
		accessSecret:   rand.Int64(),
		accessLockTime: time.Now(),
		lastClient:     clientName,
	}
	locks.locks[lockName] = lock
	stats.MasterAdminLock.WithLabelValues(clientName).Set(1)
	return lock.accessLockTime, lock.accessSecret
}

func (locks *AdminLocks) deleteLockLocked(lockName string) {
	if adminLock, found := locks.locks[lockName]; found {
		stats.MasterAdminLock.DeleteLabelValues(adminLock.lastClient)
		delete(locks.locks, lockName)
	}
}

func (ms *MasterServer) LeaseAdminToken(ctx context.Context, req *master_pb.LeaseAdminTokenRequest) (*master_pb.LeaseAdminTokenResponse, error) {
	resp := &master_pb.LeaseAdminTokenResponse{}

	if !ms.Topo.IsLeader() {
		return resp, raft.NotLeaderError
	}

	ms.adminLocks.Lock()
	defer ms.adminLocks.Unlock()
	if lastClient, lastMessage, isLocked := ms.adminLocks.lockStatusLocked(req.LockName); isLocked {
		glog.V(4).Infof("LeaseAdminToken %v", lastClient)
		if req.PreviousToken != 0 && ms.adminLocks.isValidTokenLocked(req.LockName, time.Unix(0, req.PreviousLockTime), req.PreviousToken) {
			// for renew
			ts, token := ms.adminLocks.generateTokenLocked(req.LockName, req.ClientName)
			resp.Token, resp.LockTsNs = token, ts.UnixNano()
			return resp, nil
		}
		// refuse since still locked
		return resp, fmt.Errorf("already locked by %v: %v", lastClient, lastMessage)
	}
	// for fresh lease request
	ts, token := ms.adminLocks.generateTokenLocked(req.LockName, req.ClientName)
	resp.Token, resp.LockTsNs = token, ts.UnixNano()
	return resp, nil
}

func (ms *MasterServer) ReleaseAdminToken(ctx context.Context, req *master_pb.ReleaseAdminTokenRequest) (*master_pb.ReleaseAdminTokenResponse, error) {
	resp := &master_pb.ReleaseAdminTokenResponse{}
	// a follower holds no lock state; answering success here would fake a
	// release while the leader keeps the lock until it expires
	if !ms.Topo.IsLeader() {
		return resp, raft.NotLeaderError
	}
	ms.adminLocks.Lock()
	defer ms.adminLocks.Unlock()
	if ms.adminLocks.isValidTokenLocked(req.LockName, time.Unix(0, req.PreviousLockTime), req.PreviousToken) {
		ms.adminLocks.deleteLockLocked(req.LockName)
	}
	return resp, nil
}

func (ms *MasterServer) GetAdminLockStatus(ctx context.Context, req *master_pb.GetAdminLockStatusRequest) (*master_pb.GetAdminLockStatusResponse, error) {
	resp := &master_pb.GetAdminLockStatusResponse{}
	if !ms.Topo.IsLeader() {
		return resp, raft.NotLeaderError
	}
	// isLocked reports the last holder even after the lease expired
	if clientName, message, isLocked := ms.adminLocks.isLocked(req.LockName); isLocked {
		resp.IsLocked = true
		resp.ClientName = clientName
		resp.Message = message
	}
	return resp, nil
}

// isKnownPingTarget reports whether target is a peer that the master has
// learned about as part of cluster membership. Restricting Ping to known
// peers avoids turning the RPC into a generic outbound dialer. The lookups
// are O(1) so the gate stays cheap on clusters with thousands of nodes.
func (ms *MasterServer) isKnownPingTarget(ctx context.Context, target string, targetType string) bool {
	addr := pb.ServerAddress(target)
	switch targetType {
	case cluster.FilerType, cluster.BrokerType, cluster.S3Type:
		return ms.Cluster.IsKnownNode(targetType, addr)
	case cluster.VolumeServerType:
		if ms.Topo != nil && ms.Topo.LookupDataNodeByAddress(addr) != nil {
			return true
		}
		// Only the leader receives volume-server heartbeats, so a follower's
		// topology is empty. Fall back to the volume-server set the master
		// learns over its own MasterClient subscription to the leader, the
		// same source the filer gate trusts.
		if ms.MasterClient != nil {
			return ms.MasterClient.HasVolumeServer(addr)
		}
		return false
	case cluster.MasterType:
		if ms.option != nil && ms.option.Master.Equals(addr) {
			return true
		}
		if ms.MasterClient != nil {
			_, ok := ms.MasterClient.ListMasterSet()[addr.ToHttpAddress()]
			return ok
		}
		return false
	}
	return false
}

func (ms *MasterServer) Ping(ctx context.Context, req *master_pb.PingRequest) (resp *master_pb.PingResponse, pingErr error) {
	resp = &master_pb.PingResponse{
		StartTimeNs: time.Now().UnixNano(),
	}
	// Empty target is a self-liveness probe and stays unauthenticated.
	if req.Target != "" && !ms.isKnownPingTarget(ctx, req.Target, req.TargetType) {
		resp.StopTimeNs = time.Now().UnixNano()
		return resp, status.Errorf(codes.InvalidArgument, "unknown ping target %s of type %s", req.Target, req.TargetType)
	}
	if req.TargetType == cluster.FilerType {
		pingErr = pb.WithFilerClient(false, 0, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{})
			if pingResp != nil {
				resp.RemoteTimeNs = pingResp.StartTimeNs
			}
			return err
		})
	}
	if req.TargetType == cluster.VolumeServerType {
		pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), ms.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
			pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
			if pingResp != nil {
				resp.RemoteTimeNs = pingResp.StartTimeNs
			}
			return err
		})
	}
	if req.TargetType == cluster.MasterType {
		pingErr = pb.WithMasterClient(context.Background(), false, pb.ServerAddress(req.Target), ms.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
			pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
			if pingResp != nil {
				resp.RemoteTimeNs = pingResp.StartTimeNs
			}
			return err
		})
	}
	if pingErr != nil {
		pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr)
	}
	resp.StopTimeNs = time.Now().UnixNano()
	return
}