SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package chunk_cache
import (
"fmt"
"path"
"slices"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
type OnDiskCacheLayer struct {
diskCaches []*ChunkCacheVolume
}
func NewOnDiskCacheLayer(dir, namePrefix string, diskSize int64, segmentCount int) *OnDiskCacheLayer {
volumeCount, volumeSize := int(diskSize/(30000*1024*1024)), int64(30000*1024*1024)
if volumeCount < segmentCount {
volumeCount, volumeSize = segmentCount, diskSize/int64(segmentCount)
}
c := &OnDiskCacheLayer{}
for i := 0; i < volumeCount; i++ {
fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i))
diskCache, err := LoadOrCreateChunkCacheVolume(fileName, volumeSize)
if err != nil {
glog.Errorf("failed to add cache %s : %v", fileName, err)
} else {
c.diskCaches = append(c.diskCaches, diskCache)
}
}
// keep newest cache to the front
slices.SortFunc(c.diskCaches, func(a, b *ChunkCacheVolume) int {
return b.lastModTime.Compare(a.lastModTime)
})
return c
}
func (c *OnDiskCacheLayer) setChunk(needleId types.NeedleId, data []byte) {
if len(c.diskCaches) == 0 {
return
}
if c.diskCaches[0].fileSize+int64(len(data)) > c.diskCaches[0].sizeLimit {
t, resetErr := c.diskCaches[len(c.diskCaches)-1].Reset()
if resetErr != nil {
glog.Errorf("failed to reset cache file %s", c.diskCaches[len(c.diskCaches)-1].fileName)
return
}
for i := len(c.diskCaches) - 1; i > 0; i-- {
c.diskCaches[i] = c.diskCaches[i-1]
}
c.diskCaches[0] = t
}
if err := c.diskCaches[0].WriteNeedle(needleId, data); err != nil {
glog.V(0).Infof("cache write %v size %d: %v", needleId, len(data), err)
}
}
func (c *OnDiskCacheLayer) readChunkAt(buffer []byte, needleId types.NeedleId, offset uint64) (n int, err error) {
for _, diskCache := range c.diskCaches {
n, err = diskCache.readNeedleSliceAt(buffer, needleId, offset)
if err == storage.ErrorNotFound {
continue
}
if err != nil {
glog.Warningf("failed to read cache file %s id %d: %v", diskCache.fileName, needleId, err)
continue
}
if n > 0 {
return
}
}
return
}
func (c *OnDiskCacheLayer) shutdown() {
for _, diskCache := range c.diskCaches {
diskCache.Shutdown()
}
}