SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package command
import (
"errors"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
)
// The write-backs of -allMounts start this many at a time, this far apart:
// each one replays the filer's log from its own offset, and a process that
// starts a hundred of them at once makes the filer replay a hundred logs
// together.
const (
allMountsStartBatch = 10
allMountsStartPause = 500 * time.Millisecond
allMountsRetryMin = time.Second
allMountsRetryMax = 30 * time.Second
// A directory mounted while this process runs, with no offset saved yet,
// waits this long for one: remote.mount saves it right after the
// mapping, and other mount tooling may save it later.
allMountsOffsetWait = 2 * time.Minute
// A directory whose write-back keeps ending at an unmount this many times
// in a row, while the live table still mounts it at the same location, is
// warned about: its replayed log holds an unmount it stops at every time.
allMountsGoneWarnAfter = 5
)
// mountSupervisor runs one write-back per mounted directory in this process.
// Each is the -dir write-back: its own subscription, offset, processor and
// retry loop, under its own client id (the filer keeps one subscription per
// client id, so a shared id would make each new follow replace the last).
//
// The supervisor only starts write-backs. A write-back ends itself when the
// mount table it receives, in log order, no longer maps its directory to its
// location, exactly where the -dir process exits: everything before that
// point is written back, nothing after it (remote.unmount's purge) is. The
// supervisor never stops a running write-back because the live table changed,
// which would cut off a write-back still behind in the log. A directory whose
// write-back ended is started again, from its saved offset, while the live
// table still mounts it (with a growing pause if it keeps ending at once).
type mountSupervisor struct {
option *RemoteSyncOptions
filerSource *source.FilerSource
// readMappings, follow, readOffset, now and warn are the seams the tests
// replace.
readMappings func() (*remote_pb.RemoteStorageMapping, error)
follow func(option *RemoteSyncOptions, dir string, watch *mountWatch) error
readOffset func(dir string) (int64, error)
now func() time.Time
warn func(format string, args ...interface{})
mu sync.Mutex
followers map[string]*mountFollower
// notBefore holds a directory whose write-back ended until its next start;
// ends counts its consecutive quick ends, for the pause.
notBefore map[string]time.Time
ends map[string]int
// firstSeen is when a mounted directory without a saved offset was first
// read in the table (allMountsOffsetWait).
firstSeen map[string]time.Time
// gone counts, per directory, the write-backs in a row that ended at an
// unmount while each was started at the same location and stored offset
// (goneStreak).
gone map[string]*goneStreak
wake chan struct{}
// clientIds hands each follow its own non-zero client id.
clientIds atomic.Int32
}
// goneStreak is a directory's run of write-backs that ended at an unmount,
// each started at loc from the stored offset; warned is set once the run has
// been reported.
type goneStreak struct {
loc *remote_pb.RemoteStorageLocation
offset int64
ends int
warned bool
}
type mountFollower struct {
stop chan struct{}
done chan struct{}
}
func newMountSupervisor(option *RemoteSyncOptions, filerSource *source.FilerSource) *mountSupervisor {
m := &mountSupervisor{
option: option,
filerSource: filerSource,
followers: map[string]*mountFollower{},
notBefore: map[string]time.Time{},
ends: map[string]int{},
firstSeen: map[string]time.Time{},
gone: map[string]*goneStreak{},
wake: make(chan struct{}, 1),
now: time.Now,
warn: glog.Warningf,
}
m.clientIds.Store(int32(time.Now().UnixNano()))
m.readMappings = func() (*remote_pb.RemoteStorageMapping, error) { return readMountMappings(option) }
m.follow = func(o *RemoteSyncOptions, dir string, watch *mountWatch) error {
return followMountedDir(o, filerSource, dir, watch)
}
m.readOffset = func(dir string) (int64, error) {
return remote_storage.GetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), dir)
}
return m
}
// run starts write-backs until quit closes (never, for the command), then
// stops every write-back and waits for them.
func (m *mountSupervisor) run(quit <-chan struct{}) {
interval := 5 * time.Second
if m.option.mountsCheck != nil && *m.option.mountsCheck > 0 {
interval = *m.option.mountsCheck
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
if err := m.reconcile(); err != nil {
glog.Errorf("filer.remote.sync -allMounts: read mount table: %v", err)
}
select {
case <-quit:
m.stopAll()
return
case <-ticker.C:
case <-m.wake:
}
}
}
// reconcile starts a write-back for every mounted directory that has none
// running, is past its pause, and has a saved offset (or waited for one).
func (m *mountSupervisor) reconcile() error {
mappings, err := m.readMappings()
if err != nil {
return err
}
now := m.now()
var starting []string
m.mu.Lock()
for dir := range m.firstSeen {
if mappings == nil || mappings.Mappings[dir] == nil {
delete(m.firstSeen, dir)
}
}
for dir := range m.gone {
if mappings == nil || mappings.Mappings[dir] == nil {
delete(m.gone, dir)
}
}
for dir := range m.notBefore {
if _, running := m.followers[dir]; !running && (mappings == nil || mappings.Mappings[dir] == nil) {
delete(m.notBefore, dir)
delete(m.ends, dir)
}
}
if mappings != nil {
for dir := range mappings.Mappings {
if _, running := m.followers[dir]; running {
continue
}
if t, paused := m.notBefore[dir]; paused && now.Before(t) {
continue
}
starting = append(starting, dir)
}
}
m.mu.Unlock()
sort.Strings(starting)
ready := starting[:0]
offsets := map[string]int64{}
for _, dir := range starting {
if settled, offset := m.offsetSettled(dir, now); settled {
ready = append(ready, dir)
offsets[dir] = offset
}
}
for i, dir := range ready {
if i > 0 && i%allMountsStartBatch == 0 {
time.Sleep(allMountsStartPause)
}
m.noteStart(dir, mappings.Mappings[dir], offsets[dir])
m.start(dir)
}
return nil
}
// noteStart keeps a directory's run of ends at an unmount going while it is
// started again at the same location from the same stored offset (an end
// saves none; a write-back that saved progress, or remote.unmount, moved
// it), and warns once when the run reaches
// allMountsGoneWarnAfter: a write-back that keeps replaying an unmount from
// its saved offset ends there each time and never writes the directory back.
// Nothing changes but the log; the operator decides (setting the directory's
// sync offset to now skips the replay).
func (m *mountSupervisor) noteStart(dir string, loc *remote_pb.RemoteStorageLocation, offset int64) {
m.mu.Lock()
defer m.mu.Unlock()
streak := m.gone[dir]
if streak == nil || !sameMountLocation(streak.loc, loc) || streak.offset != offset {
m.gone[dir] = &goneStreak{loc: loc, offset: offset}
return
}
if streak.ends >= allMountsGoneWarnAfter && !streak.warned {
streak.warned = true
m.warn("filer.remote.sync -allMounts: the write-back of %s has ended %d times in a row at an unmount in its replayed log while the mount table still mounts it at %+v; it is not writing the directory back. Set its sync offset to now to skip the replay", dir, streak.ends, loc)
}
}
// offsetSettled reports whether a directory may start, and its stored
// offset: it has a saved offset, or has waited allMountsOffsetWait for one.
// An unreadable offset waits too (the follow could not start from it
// anyway).
func (m *mountSupervisor) offsetSettled(dir string, now time.Time) (bool, int64) {
offset, err := m.readOffset(dir)
m.mu.Lock()
defer m.mu.Unlock()
if err == nil && offset > 0 {
delete(m.firstSeen, dir)
return true, offset
}
seen, ok := m.firstSeen[dir]
if !ok {
m.firstSeen[dir] = now
seen = now
}
if err != nil {
glog.Errorf("filer.remote.sync -allMounts: read sync offset of %s: %v", dir, err)
return false, 0
}
if now.Sub(seen) >= allMountsOffsetWait {
glog.V(0).Infof("filer.remote.sync -allMounts: %s has no saved offset after %v; starting from its creation", dir, allMountsOffsetWait)
delete(m.firstSeen, dir)
return true, 0
}
return false, 0
}
func (m *mountSupervisor) poke() {
select {
case m.wake <- struct{}{}:
default:
}
}
func (m *mountSupervisor) start(dir string) {
f := &mountFollower{stop: make(chan struct{}), done: make(chan struct{})}
m.mu.Lock()
m.followers[dir] = f
m.mu.Unlock()
option := *m.option
option.clientId = m.clientIds.Add(1)
if option.clientId == 0 { // the filer does not register client id 0
option.clientId = m.clientIds.Add(1)
}
option.clientEpoch = 0
watch := &mountWatch{stop: f.stop}
go func() {
defer close(f.done)
glog.V(0).Infof("filer.remote.sync -allMounts: synchronize %s to remote storage", dir)
began := m.now()
wait := allMountsRetryMin
for {
attempt := time.Now()
err := m.follow(&option, dir, watch)
if errors.Is(err, errFollowStopped) || isClosed(f.stop) {
m.forget(dir, f)
return
}
if errors.Is(err, errMountGone) {
m.mu.Lock()
if streak := m.gone[dir]; streak != nil {
streak.ends++
}
m.mu.Unlock()
// ended where the -dir process exits; the next reconcile
// starts it again if the live table still mounts it
m.ended(dir, f, m.now().Sub(began))
return
}
if err != nil {
glog.Errorf("synchronize %s: %v", dir, err)
}
// no stream is running here: a directory the live table no longer
// mounts is retired, and started again if it is mounted again
if !m.stillMounted(dir) {
m.forget(dir, f)
return
}
if time.Since(attempt) > allMountsRetryMax {
wait = allMountsRetryMin
}
select {
case <-f.stop:
m.forget(dir, f)
return
case <-time.After(wait):
}
if wait *= 2; wait > allMountsRetryMax {
wait = allMountsRetryMax
}
}
}()
}
// ended records a write-back that ended itself: its directory waits before
// its next start, longer each time it ends again within a minute.
func (m *mountSupervisor) ended(dir string, f *mountFollower, ran time.Duration) {
m.mu.Lock()
if m.followers[dir] == f {
delete(m.followers, dir)
}
if ran > time.Minute {
m.ends[dir] = 0
}
pause := allMountsRetryMin << m.ends[dir]
if pause > allMountsRetryMax || pause <= 0 {
pause = allMountsRetryMax
} else {
m.ends[dir]++
}
m.notBefore[dir] = m.now().Add(pause)
m.mu.Unlock()
m.poke()
}
// stillMounted reports whether the live table maps dir; an unreadable table
// counts as mapped.
func (m *mountSupervisor) stillMounted(dir string) bool {
mappings, err := m.readMappings()
return err != nil || (mappings != nil && mappings.Mappings[dir] != nil)
}
func (m *mountSupervisor) forget(dir string, f *mountFollower) {
m.mu.Lock()
if m.followers[dir] == f {
delete(m.followers, dir)
}
m.mu.Unlock()
}
func (m *mountSupervisor) stopAll() {
m.mu.Lock()
var all []*mountFollower
for _, f := range m.followers {
close(f.stop)
all = append(all, f)
}
m.mu.Unlock()
for _, f := range all {
<-f.done
}
}