SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package mount
import (
"context"
"fmt"
"net"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mount_peer_pb"
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
"github.com/seaweedfs/seaweedfs/weed/util/mem"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// fetchChunkStreamSize is the frame size used when server-streaming a chunk's
// bytes back to a peer. 1 MiB is well above gRPC's 32 KiB preferred frame but
// comfortably under the default 4 MiB message cap, so each Recv on the client
// returns quickly and the chunk is assembled with ~16 Recv calls for typical
// 16 MiB chunks.
const fetchChunkStreamSize = 1 * 1024 * 1024
// maxFetchChunkBytes caps the size of a single FetchChunk read buffer.
// The caller's expected_size is untrusted — a misbehaving peer could
// claim 1 TiB and OOM the server. 64 MiB is well above any realistic
// chunk (SeaweedFS defaults to 16 MiB, filer manifests cap at 32 MiB);
// anything larger is treated as invalid input.
const maxFetchChunkBytes = 64 * 1024 * 1024
// PeerGrpcServer is the single mount-to-mount gRPC endpoint. It serves:
// - ChunkAnnounce / ChunkLookup — the tier-2 directory RPCs, populated
// by inbound announces and queried by inbound lookups. Each handler
// is HRW-gated on the caller-side seed view.
// - FetchChunk (server stream) — serves bytes from the local
// chunk_cache to peers. Replaces the earlier HTTP-only peer-serve
// endpoint: one port, one authentication path, one connection pool.
//
// Transport credentials (TLS/mTLS) are injected via serverOpts. When the
// caller passes nothing the server runs plaintext — only appropriate on
// a trusted single-host test cluster. Production wiring passes
// security.LoadServerTLS from security.toml so cross-host traffic is
// authenticated + encrypted.
type PeerGrpcServer struct {
mount_peer_pb.UnimplementedMountPeerServer
dir *PeerDirectory
cache chunk_cache.ChunkCache
ownerFor func(fid string) string // HRW owner predicate on current seeds
selfAddr string
serverOpts []grpc.ServerOption
grpcS *grpc.Server
listener net.Listener
stopped bool
}
// NewPeerGrpcServer constructs the server. cache is the local chunk_cache
// (used to serve FetchChunk); dir is the local directory shard (used to
// answer ChunkAnnounce / ChunkLookup); ownerFor returns the HRW owner of a
// fid on the current seed view; serverOpts are forwarded to the underlying
// gRPC server (use grpc.Creds(...) for TLS/mTLS).
func NewPeerGrpcServer(cache chunk_cache.ChunkCache, dir *PeerDirectory, ownerFor func(fid string) string, selfAddr string, serverOpts ...grpc.ServerOption) *PeerGrpcServer {
return &PeerGrpcServer{
cache: cache,
dir: dir,
ownerFor: ownerFor,
selfAddr: selfAddr,
serverOpts: serverOpts,
}
}
// Start binds a TCP listener at addr and registers the MountPeer service.
func (s *PeerGrpcServer) Start(addr string) error {
ln, err := net.Listen("tcp", addr)
if err != nil {
return fmt.Errorf("peer grpc listen %s: %w", addr, err)
}
s.listener = ln
s.grpcS = pb.NewGrpcServer(s.serverOpts...)
mount_peer_pb.RegisterMountPeerServer(s.grpcS, s)
go func() {
if err := s.grpcS.Serve(ln); err != nil && err != grpc.ErrServerStopped {
glog.Warningf("peer-grpc terminated: %v", err)
}
}()
glog.V(0).Infof("peer-grpc listening on %s", ln.Addr())
return nil
}
// Stop halts the gRPC server without waiting for in-flight streams.
func (s *PeerGrpcServer) Stop() {
if s.stopped {
return
}
s.stopped = true
if s.grpcS != nil {
s.grpcS.Stop()
}
}
// Addr returns the bound address (useful when the caller used ":0").
func (s *PeerGrpcServer) Addr() string {
if s.listener == nil {
return ""
}
return s.listener.Addr().String()
}
// SelfAddr returns the advertise address this server was constructed with —
// the identity the fetcher compares against to avoid dialing itself.
func (s *PeerGrpcServer) SelfAddr() string { return s.selfAddr }
// ChunkAnnounce accepts holder entries for fids this mount owns; rejects
// others so the caller can retry against the correct owner.
func (s *PeerGrpcServer) ChunkAnnounce(ctx context.Context, req *mount_peer_pb.ChunkAnnounceRequest) (*mount_peer_pb.ChunkAnnounceResponse, error) {
ttl := time.Duration(req.TtlSeconds) * time.Second
res := s.dir.Announce(req.PeerAddr, req.DataCenter, req.Rack, req.FileIds, ttl, s.ownerPredicate)
return &mount_peer_pb.ChunkAnnounceResponse{
RejectedFileIds: res.Rejected,
}, nil
}
// ChunkLookup returns known holders for each requested fid in LRU order.
func (s *PeerGrpcServer) ChunkLookup(ctx context.Context, req *mount_peer_pb.ChunkLookupRequest) (*mount_peer_pb.ChunkLookupResponse, error) {
res := s.dir.Lookup(req.FileIds, s.ownerPredicate)
resp := &mount_peer_pb.ChunkLookupResponse{
PeersByFid: make(map[string]*mount_peer_pb.PeerSet, len(res.PeersByFid)),
NotOwnerFileIds: res.NotOwnerFids,
}
for fid, holders := range res.PeersByFid {
peers := &mount_peer_pb.PeerSet{}
for _, h := range holders {
peers.Peers = append(peers.Peers, &mount_peer_pb.PeerInfo{
PeerAddr: h.PeerAddr,
DataCenter: h.DataCenter,
Rack: h.Rack,
})
}
resp.PeersByFid[fid] = peers
}
return resp, nil
}
// FetchChunk streams bytes of a cached chunk back to the caller. Missing
// fid → gRPC NOT_FOUND. Bytes are framed at fetchChunkStreamSize so gRPC's
// default 4 MiB message cap does not constrain chunk size.
func (s *PeerGrpcServer) FetchChunk(req *mount_peer_pb.FetchChunkRequest, stream mount_peer_pb.MountPeer_FetchChunkServer) error {
if s.cache == nil {
return status.Error(codes.Unavailable, "chunk cache not configured")
}
fid := req.FileId
if fid == "" {
return status.Error(codes.InvalidArgument, "missing file_id")
}
// Size the read buffer to the caller-reported chunk length. The
// TieredChunkCache.ReadChunkAt wrapper only returns success when n
// equals len(data), so a buffer larger than the actual stored chunk
// (e.g. an 8 MiB buffer for a 2 MiB chunk) makes every read look
// like a miss. Client sends expected_size from FileChunk.Size so we
// allocate exactly the right amount. Fall back to the max-part-size
// when the caller left it zero (older clients) and hope the chunk
// is that big.
//
// expected_size is untrusted — cap at maxFetchChunkBytes to prevent
// a misbehaving peer from requesting an OOM-sized allocation.
if req.ExpectedSize > maxFetchChunkBytes {
return status.Errorf(codes.InvalidArgument,
"expected_size %d exceeds max %d", req.ExpectedSize, maxFetchChunkBytes)
}
if req.Length > maxFetchChunkBytes {
return status.Errorf(codes.InvalidArgument,
"length %d exceeds max %d", req.Length, maxFetchChunkBytes)
}
readSize := int(req.ExpectedSize)
if readSize <= 0 {
max := s.cache.GetMaxFilePartSizeInCache()
if max == 0 {
max = 8 * 1024 * 1024
}
readSize = int(max)
}
// mem.Allocate rounds up to the nearest power-of-2 slot backed by a
// shared sync.Pool; avoids an allocation per FetchChunk call.
buf := mem.Allocate(readSize)
defer mem.Free(buf)
n, err := s.cache.ReadChunkAt(buf, fid, 0)
if err != nil || n <= 0 {
return status.Errorf(codes.NotFound, "fid %s not cached", fid)
}
// Apply optional offset / length range to the whole-chunk buffer.
// Default (both 0) is a full-chunk transfer starting at byte 0,
// which is what the fetcher uses today. Non-default is reserved
// for future range-read callers.
lo := int(req.Offset)
if lo < 0 || lo > n {
return status.Errorf(codes.OutOfRange, "offset %d outside chunk length %d", lo, n)
}
hi := n
if req.Length > 0 {
hi = lo + int(req.Length)
if hi > n {
hi = n
}
}
for off := lo; off < hi; off += fetchChunkStreamSize {
end := off + fetchChunkStreamSize
if end > hi {
end = hi
}
if sendErr := stream.Send(&mount_peer_pb.FetchChunkResponse{
Data: buf[off:end],
}); sendErr != nil {
return sendErr
}
}
return nil
}
func (s *PeerGrpcServer) ownerPredicate(fid string) bool {
if s.ownerFor == nil {
return true // no HRW configured → accept all (single-mount mode)
}
return s.ownerFor(fid) == s.selfAddr
}
// peerDirectorySweepInterval is how often the mount evicts expired
// directory entries. Lookup no longer deletes inline (it takes only an
// RLock), so this sweeper is the sole memory reclamation path.
const peerDirectorySweepInterval = 60 * time.Second
// runPeerDirectorySweeper runs until stopCh closes. Stopping it on
// unmount prevents the goroutine from outliving the WFS, which matters
// for tests and for any embedded use of NewSeaweedFileSystem where the
// filesystem object is recreated without a process exit.
func (wfs *WFS) runPeerDirectorySweeper(stopCh <-chan struct{}) {
ticker := time.NewTicker(peerDirectorySweepInterval)
defer ticker.Stop()
for {
select {
case <-stopCh:
return
case <-ticker.C:
dir := wfs.peerDirectory
if dir == nil {
return
}
if evicted := dir.Sweep(); evicted > 0 {
glog.V(2).Infof("peer directory: evicted %d expired entries", evicted)
}
}
}
}