SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package weed_server
import (
"context"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
const (
remoteCacheEvictInterval = 30 * time.Second
remoteCacheEvictMinAge = time.Minute
remoteCacheVacuumCooldown = time.Minute
remoteCacheMasterRpcTime = 30 * time.Second
remoteCachePendingVidAttempts = 10
)
// uncacheRemoteEntry drops the local chunks of one remote-mounted entry, the
// same state transition remote.uncache applies through UpdateEntry. Cleared
// chunks go to the deletion queue and are reclaimed by the next compaction.
// When vids is set, only entries holding chunks on those volumes count toward
// the freed bytes, and entries contributing nothing are left untouched.
func (fs *FilerServer) uncacheRemoteEntry(ctx context.Context, fullPath util.FullPath, minCacheAge time.Duration, vids map[uint32]struct{}) (freedBytes int64, err error) {
pathLock := fs.entryLockTable.AcquireLock("uncacheRemoteEntry", fullPath, util.ExclusiveLock)
defer fs.entryLockTable.ReleaseLock(fullPath, pathLock)
current, err := fs.filer.FindEntry(ctx, fullPath)
if err != nil {
return 0, err
}
if !filer.IsEvictableRemoteEntry(current) {
return 0, nil
}
if time.Since(time.Unix(0, current.Remote.LastLocalSyncTsNs)) < minCacheAge {
return 0, nil
}
freedBytes = remoteEntryBytesOnVids(current, vids)
if freedBytes == 0 {
return 0, nil
}
newEntry := current.ShallowClone()
newEntry.Chunks = nil
newEntry.Remote = proto.Clone(current.Remote).(*filer_pb.RemoteEntry)
newEntry.Remote.LastLocalSyncTsNs = 0
if err := fs.filer.CreateEntry(ctx, newEntry, current, false, false, nil, true, fs.filer.MaxFilenameLength); err != nil {
return 0, err
}
fileIds := make([]string, 0, len(current.Chunks))
for _, chunk := range current.Chunks {
fileIds = append(fileIds, chunk.GetFileIdString())
}
fs.notePendingRemoteCacheVids(fileIds)
stats.RemoteCacheEvictedCounter.Inc()
glog.V(1).InfofCtx(ctx, "uncacheRemoteEntry %s freed %d bytes", fullPath, freedBytes)
return freedBytes, nil
}
// remoteEntryBytesOnVids sums the entry's chunk bytes on the given volumes; a
// nil set counts the whole object.
func remoteEntryBytesOnVids(entry *filer.Entry, vids map[uint32]struct{}) int64 {
if vids == nil {
return int64(entry.Size())
}
var bytes int64
for _, chunk := range entry.Chunks {
fid, err := needle.ParseFileIdFromString(chunk.GetFileIdString())
if err != nil {
continue
}
if _, ok := vids[uint32(fid.VolumeId)]; ok {
bytes += int64(chunk.Size)
}
}
return bytes
}
// evictRemoteCachedEntries drops local chunks of remote-mounted entries
// oldest-cached first until bytesNeeded is met or candidates run out. The
// first pass honors a minimum cache age so a just-fetched hot object is not
// dropped under a reader; when aged candidates cannot cover the request a
// second pass accepts any synchronized cached entry. When pressuredVids is
// set, only bytes on those volumes count and entries elsewhere are skipped.
func (fs *FilerServer) evictRemoteCachedEntries(ctx context.Context, bytesNeeded int64, pressuredVids map[uint32]struct{}) (freed int64) {
if fs.filer.RemoteStorage == nil {
return 0
}
mounts := fs.filer.RemoteStorage.MountedDirectories()
for _, minCacheAge := range []time.Duration{remoteCacheEvictMinAge, 0} {
for _, entry := range fs.filer.ListEvictableRemoteEntries(ctx, mounts, minCacheAge) {
if bytesNeeded > 0 && freed >= bytesNeeded {
return freed
}
n, err := fs.uncacheRemoteEntry(ctx, entry.FullPath, minCacheAge, pressuredVids)
if err != nil {
glog.WarningfCtx(ctx, "evict remote cache %s: %v", entry.FullPath, err)
continue
}
freed += n
}
}
return freed
}
// remoteCacheDiskPressure reports per-disk usage across the cluster: the total
// bytes to reclaim and the volumes hosted on disks over the eviction threshold.
func (fs *FilerServer) remoteCacheDiskPressure(ctx context.Context) (bytesToFree int64, pressuredVids map[uint32]struct{}, over bool) {
threshold := fs.option.RemoteCacheEvictThreshold
if threshold <= 0 {
return 0, nil, false
}
pressuredVids = make(map[uint32]struct{})
rpcCtx, cancel := context.WithTimeout(ctx, remoteCacheMasterRpcTime)
defer cancel()
err := fs.filer.MasterClient.WithClient(rpcCtx, false, func(client master_pb.SeaweedClient) error {
resp, err := client.VolumeList(rpcCtx, &master_pb.VolumeListRequest{})
if err != nil {
return err
}
for _, dc := range resp.TopologyInfo.DataCenterInfos {
for _, rack := range dc.RackInfos {
for _, dn := range rack.DataNodeInfos {
for _, disk := range dn.DiskInfos {
for _, pd := range disk.SplitByPhysicalDisk() {
if pd.DiskTotalBytes == 0 {
continue
}
used := pd.DiskTotalBytes - pd.DiskFreeBytes
if float64(used) < float64(pd.DiskTotalBytes)*threshold {
continue
}
over = true
bytesToFree += int64(used) - int64(float64(pd.DiskTotalBytes)*threshold*0.95)
for _, vi := range pd.VolumeInfos {
pressuredVids[vi.Id] = struct{}{}
}
}
}
}
}
}
return nil
})
if err != nil {
glog.WarningfCtx(ctx, "remote cache disk pressure check: %v", err)
return 0, nil, false
}
return bytesToFree, pressuredVids, over
}
// flushAndVacuumRemoteCacheVolumes forces the deletion queue down to the volume
// servers so fresh tombstones land, then compacts the volumes carrying them.
// The flush survives shutdown cancellation: an interrupted delete would be
// requeued into the in-memory retry queue that exits with the process,
// stranding bytes whose metadata the eviction already dropped.
func (fs *FilerServer) flushAndVacuumRemoteCacheVolumes(ctx context.Context) {
if len(fs.pendingRemoteCacheVids()) == 0 {
return
}
flushCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), remoteCacheMasterRpcTime)
fs.filer.FlushFileIdDeletionQueue(flushCtx, filer.LookupByMasterClientFn(fs.filer.MasterClient))
cancel()
fs.vacuumPendingRemoteCacheVids(ctx)
}
// vacuumPendingRemoteCacheVids compacts every volume still owed a vacuum. The
// master does not report whether a request actually compacted, so a vid keeps
// roughly ten minutes of attempts: tombstones that land late or compaction
// losses to another vacuum are retried by later passes instead of stranded.
func (fs *FilerServer) vacuumPendingRemoteCacheVids(ctx context.Context) {
pending := fs.pendingRemoteCacheVids()
if len(pending) == 0 {
return
}
if last := fs.remoteCacheLastVacuum.Load(); last != nil && time.Since(*last) < remoteCacheVacuumCooldown {
return
}
now := time.Now()
fs.remoteCacheLastVacuum.Store(&now)
if err := fs.filer.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error {
for vid := range pending {
vCtx, cancel := context.WithTimeout(ctx, remoteCacheMasterRpcTime)
_, err := client.VacuumVolume(vCtx, &master_pb.VacuumVolumeRequest{VolumeId: vid})
cancel()
if err != nil {
glog.WarningfCtx(ctx, "remote cache vacuum volume %d: %v", vid, err)
continue
}
fs.completePendingRemoteCacheVid(vid)
}
return nil
}); err != nil {
glog.WarningfCtx(ctx, "remote cache vacuum: %v", err)
}
}
func (fs *FilerServer) notePendingRemoteCacheVids(fileIds []string) {
fs.remoteCachePendingVidsMu.Lock()
defer fs.remoteCachePendingVidsMu.Unlock()
if fs.remoteCachePendingVids == nil {
fs.remoteCachePendingVids = make(map[uint32]int)
}
for _, fid := range fileIds {
if parsed, err := needle.ParseFileIdFromString(fid); err == nil {
fs.remoteCachePendingVids[uint32(parsed.VolumeId)] = remoteCachePendingVidAttempts
}
}
}
func (fs *FilerServer) pendingRemoteCacheVids() map[uint32]struct{} {
fs.remoteCachePendingVidsMu.Lock()
defer fs.remoteCachePendingVidsMu.Unlock()
out := make(map[uint32]struct{}, len(fs.remoteCachePendingVids))
for vid := range fs.remoteCachePendingVids {
out[vid] = struct{}{}
}
return out
}
func (fs *FilerServer) completePendingRemoteCacheVid(vid uint32) {
fs.remoteCachePendingVidsMu.Lock()
defer fs.remoteCachePendingVidsMu.Unlock()
if fs.remoteCachePendingVids[vid] <= 1 {
delete(fs.remoteCachePendingVids, vid)
} else {
fs.remoteCachePendingVids[vid]--
}
}
// reclaimRemoteCacheSpace evicts remote-cached content and compacts volumes to
// release disk space under capacity pressure. A pass already in flight is
// enough; callers that would queue behind it just fall back to remote reads.
func (fs *FilerServer) reclaimRemoteCacheSpace(ctx context.Context, bytesNeeded int64, pressuredVids map[uint32]struct{}) {
if !fs.remoteCacheEvictMu.TryLock() {
return
}
defer fs.remoteCacheEvictMu.Unlock()
freed := fs.evictRemoteCachedEntries(ctx, bytesNeeded, pressuredVids)
if freed > 0 {
glog.V(0).InfofCtx(ctx, "remote cache eviction freed %d bytes", freed)
}
fs.flushAndVacuumRemoteCacheVolumes(ctx)
}
func isRemoteCacheCapacityError(err error) bool {
msg := err.Error()
return strings.Contains(msg, "writable volumes") ||
strings.Contains(msg, "free volumes") ||
strings.Contains(msg, "no space left") ||
strings.Contains(msg, "out of space")
}
// runRemoteCacheEviction periodically evicts remote-cached entries once any
// disk crosses the configured usage threshold, with a vacuum pass to reclaim
// the deleted chunks.
func (fs *FilerServer) evictCtx() context.Context {
if fs.remoteCacheEvictCtx == nil {
return context.Background()
}
return fs.remoteCacheEvictCtx
}
func (fs *FilerServer) runRemoteCacheEviction() {
if fs.option.RemoteCacheEvictThreshold <= 0 || fs.remoteCacheEvictCtx == nil {
return
}
ctx := fs.remoteCacheEvictCtx
ticker := time.NewTicker(remoteCacheEvictInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
if fs.remoteCacheEvictMu.TryLock() {
fs.vacuumPendingRemoteCacheVids(ctx)
fs.remoteCacheEvictMu.Unlock()
}
if fs.filer.RemoteStorage == nil || len(fs.filer.RemoteStorage.MountedDirectories()) == 0 {
continue
}
bytesToFree, pressuredVids, over := fs.remoteCacheDiskPressure(ctx)
if !over {
continue
}
glog.V(0).Infof("remote cache: disk usage over %.0f%%, evicting %d bytes", fs.option.RemoteCacheEvictThreshold*100, bytesToFree)
fs.reclaimRemoteCacheSpace(ctx, bytesToFree, pressuredVids)
}
}