Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

5.6 KB
Raw Blame
package filer

import (
	"bytes"
	"fmt"
	"os"
	"sync"
	"time"

	"google.golang.org/protobuf/proto"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)

// pbEntryPool reduces allocations in EncodeAttributesAndChunks and DecodeAttributesAndChunks
// which are called on every filer store operation
var pbEntryPool = sync.Pool{
	New: func() interface{} {
		return &filer_pb.Entry{
			Attributes: &filer_pb.FuseAttributes{}, // Pre-allocate attributes
		}
	},
}

// resetPbEntry clears a protobuf Entry for reuse
func resetPbEntry(e *filer_pb.Entry) {
	// Use struct assignment to clear all fields including protobuf internal fields
	// (unknownFields, sizeCache) that field-by-field reset would miss
	attrs := e.Attributes
	*e = filer_pb.Entry{}
	if attrs == nil {
		attrs = &filer_pb.FuseAttributes{}
	} else {
		resetFuseAttributes(attrs)
	}
	e.Attributes = attrs
}

// resetFuseAttributes clears FuseAttributes for reuse
func resetFuseAttributes(a *filer_pb.FuseAttributes) {
	// Use struct assignment to clear all fields including protobuf internal fields
	*a = filer_pb.FuseAttributes{}
}

func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
	message := pbEntryPool.Get().(*filer_pb.Entry)
	defer func() {
		resetPbEntry(message)
		pbEntryPool.Put(message)
	}()

	entry.ToExistingProtoEntry(message)

	data, err := proto.Marshal(message)
	if err != nil {
		return nil, err
	}

	// Copy the data to a new slice since proto.Marshal may return a slice
	// that shares memory with the message (not guaranteed to be a copy)
	return append([]byte(nil), data...), nil
}

func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error {
	message := pbEntryPool.Get().(*filer_pb.Entry)
	defer func() {
		resetPbEntry(message)
		pbEntryPool.Put(message)
	}()

	if err := proto.Unmarshal(blob, message); err != nil {
		return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err)
	}

	FromPbEntryToExistingEntry(message, entry)

	return nil
}

func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {

	return &filer_pb.FuseAttributes{
		Crtime:        entry.Attr.Crtime.Unix(),
		Mtime:         entry.Attr.Mtime.Unix(),
		MtimeNs:       int32(entry.Attr.Mtime.Nanosecond()),
		Ctime:         entry.Attr.Ctime.Unix(),
		CtimeNs:       int32(entry.Attr.Ctime.Nanosecond()),
		Atime:         atimeSecondsForPb(entry.Attr),
		AtimeNs:       atimeNanosForPb(entry.Attr),
		FileMode:      uint32(entry.Attr.Mode),
		Uid:           entry.Uid,
		Gid:           entry.Gid,
		Mime:          entry.Mime,
		TtlSec:        entry.Attr.TtlSec,
		UserName:      entry.Attr.UserName,
		GroupName:     entry.Attr.GroupNames,
		SymlinkTarget: entry.Attr.SymlinkTarget,
		Md5:           entry.Attr.Md5,
		FileSize:      entry.Attr.FileSize,
		Rdev:          entry.Attr.Rdev,
		Inode:         entry.Attr.Inode,
	}
}

func atimeSecondsForPb(attr Attr) int64 {
	if attr.Atime.IsZero() {
		return 0
	}
	return attr.Atime.Unix()
}

func atimeNanosForPb(attr Attr) int32 {
	if attr.Atime.IsZero() {
		return 0
	}
	return int32(attr.Atime.Nanosecond())
}

// EntryAttributeToExistingPb fills an existing FuseAttributes to avoid allocation.
// Safe to call with nil attr (will return early without populating).
func EntryAttributeToExistingPb(entry *Entry, attr *filer_pb.FuseAttributes) {
	if attr == nil {
		return
	}
	attr.Crtime = entry.Attr.Crtime.Unix()
	attr.Mtime = entry.Attr.Mtime.Unix()
	attr.MtimeNs = int32(entry.Attr.Mtime.Nanosecond())
	attr.Ctime = entry.Attr.Ctime.Unix()
	attr.CtimeNs = int32(entry.Attr.Ctime.Nanosecond())
	attr.Atime = atimeSecondsForPb(entry.Attr)
	attr.AtimeNs = atimeNanosForPb(entry.Attr)
	attr.FileMode = uint32(entry.Attr.Mode)
	attr.Uid = entry.Uid
	attr.Gid = entry.Gid
	attr.Mime = entry.Mime
	attr.TtlSec = entry.Attr.TtlSec
	attr.UserName = entry.Attr.UserName
	attr.GroupName = entry.Attr.GroupNames
	attr.SymlinkTarget = entry.Attr.SymlinkTarget
	attr.Md5 = entry.Attr.Md5
	attr.FileSize = entry.Attr.FileSize
	attr.Rdev = entry.Attr.Rdev
	attr.Inode = entry.Attr.Inode
}

func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {

	t := Attr{}

	if attr == nil {
		return t
	}

	t.Crtime = time.Unix(attr.Crtime, 0)
	t.Mtime = time.Unix(attr.Mtime, int64(attr.MtimeNs))
	if attr.Ctime != 0 {
		t.Ctime = time.Unix(attr.Ctime, int64(attr.CtimeNs))
	} else {
		t.Ctime = t.Mtime
	}
	if attr.Atime != 0 || attr.AtimeNs != 0 {
		t.Atime = time.Unix(attr.Atime, int64(attr.AtimeNs))
	} else {
		t.Atime = t.Mtime
	}
	t.Mode = os.FileMode(attr.FileMode)
	t.Uid = attr.Uid
	t.Gid = attr.Gid
	t.Mime = attr.Mime
	t.TtlSec = attr.TtlSec
	t.UserName = attr.UserName
	t.GroupNames = attr.GroupName
	t.SymlinkTarget = attr.SymlinkTarget
	t.Md5 = attr.Md5
	t.FileSize = attr.FileSize
	t.Rdev = attr.Rdev
	t.Inode = attr.Inode

	return t
}

func EqualEntry(a, b *Entry) bool {
	if a == b {
		return true
	}
	if a == nil && b != nil || a != nil && b == nil {
		return false
	}
	if !proto.Equal(EntryAttributeToPb(a), EntryAttributeToPb(b)) {
		return false
	}
	if len(a.Chunks) != len(b.Chunks) {
		return false
	}

	if !eq(a.Extended, b.Extended) {
		return false
	}

	if !bytes.Equal(a.Md5, b.Md5) {
		return false
	}

	for i := 0; i < len(a.Chunks); i++ {
		if !proto.Equal(a.Chunks[i], b.Chunks[i]) {
			return false
		}
	}

	if !bytes.Equal(a.HardLinkId, b.HardLinkId) {
		return false
	}
	if a.HardLinkCounter != b.HardLinkCounter {
		return false
	}
	if !bytes.Equal(a.Content, b.Content) {
		return false
	}
	if !proto.Equal(a.Remote, b.Remote) {
		return false
	}
	if a.Quota != b.Quota {
		return false
	}
	return true
}

func eq(a, b map[string][]byte) bool {
	if len(a) != len(b) {
		return false
	}

	for k, v := range a {
		if w, ok := b[k]; !ok || !bytes.Equal(v, w) {
			return false
		}
	}

	return true
}