Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

20.9 KB
Raw Blame
package cluster

import (
	"context"
	"fmt"
	"slices"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"google.golang.org/grpc"
)

type LockClient struct {
	grpcDialOption  grpc.DialOption
	maxLockDuration time.Duration
	sleepDuration   time.Duration
	seedFiler       pb.ServerAddress

	// ring is an optional client-side view of the filer lock hash ring. When
	// populated, a new lock starts at the key's primary filer instead of the
	// seed filer, avoiding the seed->primary forward hop. A stale view stays
	// correct: the filer forwards to the real primary as a fallback.
	ringMu      sync.RWMutex
	ring        *lock_manager.HashRing
	ringServers []pb.ServerAddress
	ringVersion int64

	// priorRing is the ring before the most recent change, kept for priorWindow so a
	// route-by-key caller can consult a just-moved key's previous owner during a
	// rebalance. Mirrors the master's lock_manager.LockRing.PriorOwner cooling-off.
	priorRing     *lock_manager.HashRing
	ringChangedAt time.Time
	priorWindow   time.Duration

	// noLockServerRetryPeriod bounds retries when every filer reports "no
	// lock server found": an empty lock ring is a systemic fault that waiting
	// on a lock holder cannot resolve, unlike ordinary contention. The bound
	// is long enough to ride out a master leader change (ring reset + fresh
	// snapshot) but short enough that a write fails instead of hanging
	// forever.
	noLockServerRetryPeriod time.Duration

	// lockRpcTimeout bounds a single DistributedLock attempt: a ring member
	// reachable from the masters but not from this filer would otherwise
	// block an attempt on the TCP connect timeout, far past
	// noLockServerRetryPeriod.
	lockRpcTimeout time.Duration
}

func NewLockClient(grpcDialOption grpc.DialOption, seedFiler pb.ServerAddress) *LockClient {
	return &LockClient{
		grpcDialOption:          grpcDialOption,
		maxLockDuration:         5 * time.Second,
		sleepDuration:           2473 * time.Millisecond,
		seedFiler:               seedFiler,
		priorWindow:             5 * time.Second,
		noLockServerRetryPeriod: 15 * time.Second,
		lockRpcTimeout:          10 * time.Second,
	}
}

// SetRing mirrors the master's LockRingUpdate so the client computes the same
// primary the filers do. A non-zero version at or below the current one is
// ignored once a ring exists, dropping reordered and redundant broadcasts;
// version 0 always applies (bootstrap).
func (lc *LockClient) SetRing(servers []pb.ServerAddress, version int64) {
	lc.ringMu.Lock()
	defer lc.ringMu.Unlock()
	if version != 0 && version <= lc.ringVersion && lc.ring != nil {
		return
	}
	lc.ringVersion = version
	sorted := slices.Clone(servers)
	slices.Sort(sorted)
	if slices.Equal(sorted, lc.ringServers) {
		return
	}
	lc.ringServers = sorted
	// Build a fresh ring (not an in-place mutation) so the outgoing ring survives as
	// priorRing with its own servers for the cooling-off window.
	newRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
	newRing.SetServers(sorted)
	if lc.ring != nil {
		lc.priorRing = lc.ring
		lc.ringChangedAt = time.Now()
	}
	lc.ring = newRing
}

// ResetRing clears only the version gate so the first update from a
// different master applies unconditionally: ring versions are per-master
// monotonic and a high version accepted from a former leader must not
// reject the new leader's snapshot. The last ring keeps routing during the
// gap rather than falling back to the seed filer, and the arriving ring
// becomes the prior ring for the cooling-off window.
func (lc *LockClient) ResetRing() {
	lc.ringMu.Lock()
	defer lc.ringMu.Unlock()
	lc.ringVersion = 0
}

// hostForKey returns the filer that should own key per the current ring view,
// falling back to the seed filer when no view has been received yet.
func (lc *LockClient) hostForKey(key string) pb.ServerAddress {
	lc.ringMu.RLock()
	defer lc.ringMu.RUnlock()
	if lc.ring == nil {
		return lc.seedFiler
	}
	if primary := lc.ring.GetPrimary(key); primary != "" {
		return primary
	}
	return lc.seedFiler
}

// PrimaryForKey returns the ring owner for key, or "" before any ring arrives.
// Unlike hostForKey it does not fall back to the seed, so a route-by-key caller
// stays on the distributed lock until the ring is known.
func (lc *LockClient) PrimaryForKey(key string) pb.ServerAddress {
	lc.ringMu.RLock()
	defer lc.ringMu.RUnlock()
	if lc.ring == nil {
		return ""
	}
	return lc.ring.GetPrimary(key)
}

// PriorOwnerForKey returns key's owner from the prior ring while a rebalance is
// within the cooling-off window and ownership actually moved, else "". Lets a
// route-by-key reader consult a just-moved key's previous owner before the new
// owner's NotFound is final (the new owner may not have replicated the key yet).
func (lc *LockClient) PriorOwnerForKey(key string) pb.ServerAddress {
	lc.ringMu.RLock()
	defer lc.ringMu.RUnlock()
	if lc.ring == nil || lc.priorRing == nil {
		return ""
	}
	if time.Since(lc.ringChangedAt) > lc.priorWindow {
		return ""
	}
	current := lc.ring.GetPrimary(key)
	prior := lc.priorRing.GetPrimary(key)
	if prior != "" && prior != current {
		return prior
	}
	return ""
}

type LiveLock struct {
	generation          int64 // fencing token from the lock server; MUST stay first so the 64-bit atomic ops stay 8-byte aligned on 32-bit ARM
	key                 string
	renewToken          string
	expireAtNs          int64
	hostFiler           pb.ServerAddress
	cancelCh            chan struct{}
	renewalDone         chan struct{} // closed when the renewal goroutine exits; nil if there is none
	grpcDialOption      grpc.DialOption
	isLocked            int32 // 0 = unlocked, 1 = locked; use atomic operations
	self                string
	lc                  *LockClient
	owner               string
	lockTTL             time.Duration
	consecutiveFailures int // Track connection failures to trigger fallback
}

// NewShortLivedLock creates a lock with a 5-second duration.
// It returns nil when the lock cannot be acquired because no lock server
// exists; ordinary contention is still waited out.
func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLock) {
	lock = &LiveLock{
		key:            key,
		hostFiler:      lc.hostForKey(key),
		cancelCh:       make(chan struct{}),
		expireAtNs:     time.Now().Add(5 * time.Second).UnixNano(),
		grpcDialOption: lc.grpcDialOption,
		self:           owner,
		lc:             lc,
	}
	if err := lock.retryUntilLocked(5*time.Second, 0); err != nil {
		glog.Warningf("create lock %s: %v", key, err)
		return nil
	}
	return
}

// NewBlockingLongLivedLock blocks until the lock is acquired, then starts a
// background renewal goroutine that keeps the lock alive. This combines the
// synchronous acquisition of NewShortLivedLock with the auto-renewal of
// StartLongLivedLock. Release with Stop().
func (lc *LockClient) NewBlockingLongLivedLock(key, owner string, lockTTL time.Duration) *LiveLock {
	lock, err := lc.NewBlockingLongLivedLockWithin(key, owner, lockTTL, 0)
	if err != nil {
		glog.Warningf("create lock %s: %v", key, err)
		return nil
	}
	return lock
}

// NewBlockingLongLivedLockWithin is NewBlockingLongLivedLock that gives up once
// another owner has held the lock for maxWait. Zero waits without bound.
func (lc *LockClient) NewBlockingLongLivedLockWithin(key, owner string, lockTTL, maxWait time.Duration) (*LiveLock, error) {
	if lockTTL == 0 {
		lockTTL = lock_manager.LiveLockTTL
	}
	lock := &LiveLock{
		key:            key,
		hostFiler:      lc.hostForKey(key),
		cancelCh:       make(chan struct{}),
		expireAtNs:     time.Now().Add(lockTTL).UnixNano(),
		grpcDialOption: lc.grpcDialOption,
		self:           owner,
		lc:             lc,
		lockTTL:        lockTTL,
	}
	if err := lock.retryUntilLocked(lockTTL, maxWait); err != nil {
		return nil, err
	}
	// Start renewal goroutine using a ticker for interruptible sleep
	lock.renewalDone = make(chan struct{})
	go func() {
		defer close(lock.renewalDone)
		renewInterval := lockTTL / 2
		ticker := time.NewTicker(renewInterval)
		defer ticker.Stop()
		for {
			select {
			case <-lock.cancelCh:
				return
			case <-ticker.C:
				if err := lock.AttemptToLock(lockTTL); err != nil {
					glog.V(0).Infof("lock renewal failed for %s: %v", key, err)
					atomic.StoreInt32(&lock.isLocked, 0)
				}
			}
		}
	}()
	return lock, nil
}

// StartLongLivedLock starts a goroutine to lock the key and returns immediately.
// lockTTL specifies how long the lock should be held. The renewal interval is
// automatically derived as lockTTL / 2 to ensure timely renewals.
func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string), lockTTL time.Duration) (lock *LiveLock) {
	lock = &LiveLock{
		key:            key,
		hostFiler:      lc.hostForKey(key),
		cancelCh:       make(chan struct{}),
		expireAtNs:     time.Now().Add(lockTTL).UnixNano(),
		grpcDialOption: lc.grpcDialOption,
		self:           owner,
		lc:             lc,
		lockTTL:        lockTTL,
	}
	if lock.lockTTL == 0 {
		lock.lockTTL = lock_manager.LiveLockTTL
	}
	lock.renewalDone = make(chan struct{})
	go func() {
		defer close(lock.renewalDone)
		renewInterval := lock.lockTTL / 2
		isLocked := false
		lockOwner := ""
		for {
			// Check for cancellation BEFORE attempting to lock to avoid race condition
			// where Stop() is called after sleep but before lock attempt
			select {
			case <-lock.cancelCh:
				return
			default:
			}

			if isLocked {
				if err := lock.AttemptToLock(lock.lockTTL); err != nil {
					glog.V(0).Infof("Lost lock %s: %v", key, err)
					isLocked = false
					atomic.StoreInt32(&lock.isLocked, 0)
				}
			} else {
				if err := lock.AttemptToLock(lock.lockTTL); err == nil {
					isLocked = true
					// Note: AttemptToLock already sets lock.isLocked atomically on success
				}
			}
			if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
				glog.V(0).Infof("Lock owner changed from %s to %s", lockOwner, lock.LockOwner())
				onLockOwnerChange(lock.LockOwner())
				lockOwner = lock.LockOwner()
			}
			// Sleep until the next attempt, but wake immediately on Stop() so
			// the goroutine exits and closes renewalDone before Stop()'s bounded
			// wait elapses. An uninterruptible sleep here (up to 5*renewInterval
			// when unlocked) can outlast that wait and break the shutdown
			// synchronization.
			sleepFor := renewInterval
			if !isLocked {
				sleepFor = 5 * renewInterval
			}
			timer := time.NewTimer(sleepFor)
			select {
			case <-lock.cancelCh:
				timer.Stop()
				return
			case <-timer.C:
			}
		}
	}()
	return
}

// retryUntilLocked blocks until the lock is acquired, polling at the steady
// short cadence that AttemptToLock already enforces on contention (~1s). It
// deliberately avoids util.RetryUntil's exponential backoff (which grows to
// several seconds): when a holder on another mount releases the lock, the
// waiter must pick it up promptly, otherwise cross-mount write handoff stalls
// long enough to time out clients.
func (lock *LiveLock) retryUntilLocked(lockDuration, maxWait time.Duration) error {
	started := time.Now()
	// The retry window bounds the attempts against one route: the clock
	// starts before the first attempt and each attempt gets what is left
	// of it. A spent window on a non-seed target falls back to the seed
	// filer — which can forward to the real owner — and that route gets
	// its own window, so a reachable fallback is always tried.
	unavailableSince := time.Now()
	// The seed route gets one fresh window per acquisition: the seed can
	// redirect back to a dead primary via LockHostMovedTo, and refilling the
	// window on every return to the seed would make that cycle unbounded.
	seedRetried := false
	var err error
	for {
		attemptTimeout := lock.lc.lockRpcTimeout
		if remaining := lock.lc.noLockServerRetryPeriod - time.Since(unavailableSince); remaining < attemptTimeout {
			if remaining <= 0 {
				if lock.hostFiler.Equals(lock.lc.seedFiler) || seedRetried {
					if err == nil {
						err = fmt.Errorf("lock %s: no lock server granted within %s", lock.key, lock.lc.noLockServerRetryPeriod)
					}
					return err
				}
				glog.V(0).Infof("LOCK: no response within %s for key=%s filer=%s, falling back to seed filer=%s",
					lock.lc.noLockServerRetryPeriod, lock.key, lock.hostFiler, lock.lc.seedFiler)
				lock.hostFiler = lock.lc.seedFiler
				seedRetried = true
				unavailableSince = time.Now()
				continue
			}
			attemptTimeout = remaining
		}
		hostBefore := lock.hostFiler
		var granted bool
		granted, err = lock.attemptToLockWithin(lockDuration, attemptTimeout)
		if err == nil {
			if granted {
				return nil
			}
			// A clean LockHostMovedTo redirect returns no error and no
			// verdict; it must still count against the window, or a redirect
			// cycle between ring-disagreeing filers retries forever.
			continue
		}
		glog.V(1).Infof("create lock %s: %v", lock.key, err)
		if strings.Contains(err.Error(), "lock already owned") {
			// Ordinary contention: a reachable server holds the lock, so
			// waiting is the point and is bounded only by maxWait.
			if maxWait > 0 && time.Since(started) >= maxWait {
				return err
			}
			unavailableSince = time.Now()
			continue
		}
		// Anything else — "no lock server found", a dead ring member refusing
		// connections — is a systemic fault waiting cannot fix. A failed
		// attempt that fell back to the seed route gets a fresh window for
		// it; a server redirect stays inside the current one.
		if !seedRetried && lock.hostFiler.Equals(lock.lc.seedFiler) && !hostBefore.Equals(lock.lc.seedFiler) {
			seedRetried = true
			unavailableSince = time.Now()
		}
	}
}

func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
	_, err := lock.attemptToLockWithin(lockDuration, lock.lc.lockRpcTimeout)
	return err
}

func (lock *LiveLock) attemptToLockWithin(lockDuration, attemptTimeout time.Duration) (granted bool, err error) {
	glog.V(4).Infof("LOCK: AttemptToLock key=%s owner=%s", lock.key, lock.self)
	errorMessage, granted, err := lock.doLock(lockDuration, attemptTimeout)
	if err != nil {
		glog.V(1).Infof("LOCK: doLock failed for key=%s: %v", lock.key, err)
		time.Sleep(time.Second)
		return false, err
	}
	if errorMessage != "" {
		if strings.Contains(errorMessage, "lock already owned") {
			glog.V(3).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
		} else {
			glog.V(2).Infof("LOCK: doLock returned error message for key=%s: %s", lock.key, errorMessage)
		}
		time.Sleep(time.Second)
		return false, fmt.Errorf("%v", errorMessage)
	}
	if !granted {
		// A bare LockHostMovedTo redirect carries no verdict: nothing was
		// acquired, so leave isLocked as it is and keep the same ~1s cadence
		// as other retries.
		time.Sleep(time.Second)
		return false, nil
	}
	if atomic.LoadInt32(&lock.isLocked) == 0 {
		// Only log when transitioning from unlocked to locked
		glog.V(1).Infof("LOCK: Successfully acquired key=%s owner=%s", lock.key, lock.self)
	}
	atomic.StoreInt32(&lock.isLocked, 1)
	return true, nil
}

func (lock *LiveLock) StopShortLivedLock() error {
	if atomic.LoadInt32(&lock.isLocked) == 0 {
		return nil
	}
	defer func() {
		atomic.StoreInt32(&lock.isLocked, 0)
	}()
	return pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		ctx, cancel := context.WithTimeout(context.Background(), lock.lc.lockRpcTimeout)
		defer cancel()
		_, err := client.DistributedUnlock(ctx, &filer_pb.UnlockRequest{
			Name:       lock.key,
			RenewToken: lock.renewToken,
		})
		return err
	})
}

// Stop stops a long-lived lock by closing the cancel channel and releasing the lock
func (lock *LiveLock) Stop() error {
	// Close the cancel channel to stop the long-lived lock goroutine
	select {
	case <-lock.cancelCh:
		// Already closed
	default:
		close(lock.cancelCh)
	}

	// Wait for the renewal goroutine to fully exit before unlocking. A renewal
	// in flight when we close cancelCh rotates renewToken on the server; if we
	// then unlock with the token we read here, the unlock fails with a token
	// mismatch and the lock lingers until its TTL expires — blocking other
	// mounts waiting on the same file. Waiting for the goroutine to return also
	// makes the renewToken read below race-free (channel close = happens-before).
	if lock.renewalDone != nil {
		select {
		case <-lock.renewalDone:
		case <-time.After(lock.lockTTL + 2*time.Second):
			// The renewal goroutine is wedged, almost certainly in a stuck
			// renewal RPC. Do not unlock here: the renewToken may be rotated
			// when that RPC finally returns, so an unlock sent now could race
			// it, be rejected on a stale token, and leave the lock lingering
			// anyway. cancelCh is closed, so the goroutine stops renewing once
			// its in-flight call returns and the lock then expires within its
			// TTL on its own.
			glog.Warningf("lock %s: renewal goroutine still running at shutdown; letting lock expire via TTL", lock.key)
			return nil
		}
	}

	// Also release the lock if held
	// Note: We intentionally don't clear renewToken here because
	// StopShortLivedLock needs it to properly unlock
	return lock.StopShortLivedLock()
}

func (lock *LiveLock) doLock(lockDuration, attemptTimeout time.Duration) (errorMessage string, granted bool, err error) {
	glog.V(4).Infof("LOCK: doLock calling DistributedLock - key=%s filer=%s owner=%s",
		lock.key, lock.hostFiler, lock.self)

	previousHostFiler := lock.hostFiler
	previousOwner := lock.owner

	err = pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		// Bound each attempt: a ring member that is reachable from the masters
		// but not from this filer would otherwise block on the connect timeout.
		ctx, cancel := context.WithTimeout(context.Background(), attemptTimeout)
		defer cancel()
		resp, err := client.DistributedLock(ctx, &filer_pb.LockRequest{
			Name:          lock.key,
			SecondsToLock: int64(lockDuration.Seconds()),
			RenewToken:    lock.renewToken,
			IsMoved:       false,
			Owner:         lock.self,
		})
		glog.V(4).Infof("LOCK: DistributedLock response - key=%s err=%v", lock.key, err)
		if err == nil && resp != nil {
			// Only a verdict replaces the token: a grant carries the new
			// token, a rejection carries an error. A bare LockHostMovedTo
			// redirect answers neither and must not drop a held token —
			// the next attempt can still renew with it on the new host.
			granted = resp.Error == "" && resp.RenewToken != ""
			if resp.RenewToken != "" || resp.Error != "" {
				lock.renewToken = resp.RenewToken
			}
			if resp.Generation > 0 {
				atomic.StoreInt64(&lock.generation, resp.Generation)
			}
			lock.consecutiveFailures = 0 // Reset failure counter on success
			glog.V(4).Infof("LOCK: Got renewToken for key=%s", lock.key)
		} else {
			// The request may have renewed the lease server-side before its
			// response was lost: the last token can still be valid, so the
			// next attempt retries with it. A definitive rejection clears it
			// through the response's empty RenewToken.
			glog.V(1).Infof("LOCK: no response for key=%s (err=%v)", lock.key, err)
		}
		if resp != nil {
			errorMessage = resp.Error
			if resp.LockHostMovedTo != "" && !pb.ServerAddress(resp.LockHostMovedTo).Equals(previousHostFiler) {
				// Only log if the host actually changed
				glog.V(2).Infof("LOCK: Host changed from %s to %s for key=%s", previousHostFiler, resp.LockHostMovedTo, lock.key)
				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
				// Don't update seedFiler - keep original for fallback
			} else if resp.LockHostMovedTo != "" {
				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
			}
			if resp.LockOwner != "" && resp.LockOwner != previousOwner {
				// Only log if the owner actually changed
				glog.V(2).Infof("LOCK: Owner changed from %s to %s for key=%s", previousOwner, resp.LockOwner, lock.key)
				lock.owner = resp.LockOwner
			} else if resp.LockOwner != "" {
				lock.owner = resp.LockOwner
			} else if previousOwner != "" {
				glog.V(2).Infof("LOCK: Owner cleared for key=%s", lock.key)
				lock.owner = ""
			}
		}
		return err
	})

	if err != nil && !lock.hostFiler.Equals(lock.lc.seedFiler) {
		lock.consecutiveFailures++
		// Fall back to seed filer after 3 consecutive connection failures
		if lock.consecutiveFailures >= 3 {
			glog.V(0).Infof("LOCK: Connection failed %d times for key=%s filer=%s, falling back to seed filer=%s",
				lock.consecutiveFailures, lock.key, lock.hostFiler, lock.lc.seedFiler)
			lock.hostFiler = lock.lc.seedFiler
			lock.consecutiveFailures = 0
		}
	}

	return
}

func (lock *LiveLock) LockOwner() string {
	return lock.owner
}

// Generation returns the fencing token for this lock.
// It increments on each fresh acquisition and stays the same on renewal.
func (lock *LiveLock) Generation() int64 {
	return atomic.LoadInt64(&lock.generation)
}

// IsLocked returns true if this instance currently holds the lock
func (lock *LiveLock) IsLocked() bool {
	return atomic.LoadInt32(&lock.isLocked) == 1
}