Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

4.3 KB
Raw Blame
package filer

import (
	"context"
	"sync"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)

const SectionSize = 2 * 1024 * 1024 * 32 // 64MiB
type SectionIndex int64
type FileChunkSection struct {
	sectionIndex     SectionIndex
	chunks           []*filer_pb.FileChunk
	visibleIntervals *IntervalList[*VisibleInterval]
	chunkViews       *IntervalList[*ChunkView]
	reader           *ChunkReadAt
	lock             sync.RWMutex
	isPrepared       bool
}

func sectionBounds(sectionIndex SectionIndex, fileSize int64) (start, stop int64) {
	start = int64(sectionIndex) * SectionSize
	stop = fileSize
	if sectionIndex < SectionIndex(fileSize/SectionSize) {
		stop = start + SectionSize
	}
	return
}

func NewFileChunkSection(si SectionIndex) *FileChunkSection {
	return &FileChunkSection{
		sectionIndex: si,
	}
}

func (section *FileChunkSection) addChunk(chunk *filer_pb.FileChunk) error {
	section.lock.Lock()
	defer section.lock.Unlock()

	start, stop := max(int64(section.sectionIndex)*SectionSize, chunk.Offset), min(((int64(section.sectionIndex)+1)*SectionSize), chunk.Offset+int64(chunk.Size))

	section.chunks = append(section.chunks, chunk)

	if section.visibleIntervals == nil {
		section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
	} else {
		MergeIntoVisibles(section.visibleIntervals, start, stop, chunk)
		garbageFileIds := FindGarbageChunks(section.visibleIntervals, start, stop)
		removeGarbageChunks(section, garbageFileIds)
	}

	if section.chunkViews != nil {
		MergeIntoChunkViews(section.chunkViews, start, stop, chunk)
	}

	return nil
}

func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]struct{}) {
	for i := 0; i < len(section.chunks); {
		t := section.chunks[i]
		length := len(section.chunks)
		if _, found := garbageFileIds[t.FileId]; found {
			if i < length-1 {
				section.chunks[i] = section.chunks[length-1]
			}
			section.chunks = section.chunks[:length-1]
		} else {
			i++
		}
	}
}

func (section *FileChunkSection) setupForRead(ctx context.Context, group *ChunkGroup, fileSize int64) {
	section.lock.Lock()
	defer section.lock.Unlock()

	sectionStart, sectionStop := sectionBounds(section.sectionIndex, fileSize)

	if section.isPrepared {
		section.reader.fileSize = fileSize
		return
	}

	if section.visibleIntervals == nil {
		section.visibleIntervals = readResolvedChunks(section.chunks, sectionStart, sectionStop)
		section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks)
		if section.reader != nil {
			_ = section.reader.Close()
			section.reader = nil
		}
	}
	if section.chunkViews == nil {
		section.chunkViews = ViewFromVisibleIntervals(section.visibleIntervals, sectionStart, sectionStop)
	}

	if section.reader == nil {
		section.reader = NewChunkReaderAtFromClient(ctx, group.readerCache, section.chunkViews, sectionStop, group.GetPrefetchCount())
	}

	section.isPrepared = true
	section.reader.fileSize = fileSize
}

func (section *FileChunkSection) readDataAt(ctx context.Context, group *ChunkGroup, fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {

	section.setupForRead(ctx, group, fileSize)
	section.lock.RLock()
	defer section.lock.RUnlock()

	return section.reader.ReadAtWithTime(ctx, buff, offset)
}

func (section *FileChunkSection) DataStartOffset(ctx context.Context, group *ChunkGroup, offset int64, fileSize int64) int64 {

	section.setupForRead(ctx, group, fileSize)
	section.lock.RLock()
	defer section.lock.RUnlock()

	for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
		visible := x.Value
		if visible.stop <= offset {
			continue
		}
		if offset < visible.start {
			return visible.start
		}
		return offset
	}
	return -1
}

func (section *FileChunkSection) NextStopOffset(ctx context.Context, group *ChunkGroup, offset int64, fileSize int64) int64 {

	section.setupForRead(ctx, group, fileSize)
	section.lock.RLock()
	defer section.lock.RUnlock()

	isAfterOffset := false
	for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
		visible := x.Value
		if !isAfterOffset {
			if visible.stop <= offset {
				continue
			}
			isAfterOffset = true
		}
		if offset < visible.start {
			return offset
		}
		// now visible.start <= offset
		if offset < visible.stop {
			offset = visible.stop
		}
	}
	return offset
}