Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

5.8 KB
Raw Blame
package filer

import (
	"os"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

type Attr struct {
	Mtime         time.Time   // time of last modification
	Crtime        time.Time   // time of creation (OS X only)
	Ctime         time.Time   // time of last inode change
	Atime         time.Time   // time of last access
	Mode          os.FileMode // file mode
	Uid           uint32      // owner uid
	Gid           uint32      // group gid
	Mime          string      // mime type
	TtlSec        int32       // ttl in seconds
	UserName      string
	GroupNames    []string
	SymlinkTarget string
	Md5           []byte
	FileSize      uint64
	Rdev          uint32
	Inode         uint64
}

func (attr Attr) IsDirectory() bool {
	return attr.Mode&os.ModeDir > 0
}

type Entry struct {
	util.FullPath

	Attr
	Extended map[string][]byte

	// the following is for files
	Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"`

	HardLinkId         HardLinkId
	HardLinkCounter    int32
	Content            []byte
	Remote             *filer_pb.RemoteEntry
	Quota              int64
	WORMEnforcedAtTsNs int64
	// CommitTsNs is the meta log event timestamp of the mutation that
	// produced this version. Zero for pre-field versions; they order by mtime.
	CommitTsNs int64
}

func (entry *Entry) Size() uint64 {
	return maxUint64(maxUint64(TotalSize(entry.GetChunks()), entry.FileSize), uint64(len(entry.Content)))
}

func (entry *Entry) Timestamp() time.Time {
	if entry.IsDirectory() {
		return entry.Crtime
	} else {
		return entry.Mtime
	}
}

func (entry *Entry) ShallowClone() *Entry {
	if entry == nil {
		return nil
	}
	newEntry := &Entry{}
	newEntry.FullPath = entry.FullPath
	newEntry.Attr = entry.Attr
	newEntry.Chunks = entry.Chunks
	newEntry.Extended = entry.Extended
	newEntry.HardLinkId = entry.HardLinkId
	newEntry.HardLinkCounter = entry.HardLinkCounter
	newEntry.Content = entry.Content
	newEntry.Remote = entry.Remote
	newEntry.Quota = entry.Quota
	newEntry.WORMEnforcedAtTsNs = entry.WORMEnforcedAtTsNs
	newEntry.CommitTsNs = entry.CommitTsNs

	return newEntry
}

func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
	if entry == nil {
		return nil
	}
	message := &filer_pb.Entry{}
	message.Name = entry.FullPath.Name()
	entry.ToExistingProtoEntry(message)
	return message
}

func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry) {
	if entry == nil {
		return
	}
	message.IsDirectory = entry.IsDirectory()
	// Reuse pre-allocated attributes if available, otherwise allocate
	if message.Attributes != nil {
		EntryAttributeToExistingPb(entry, message.Attributes)
	} else {
		message.Attributes = EntryAttributeToPb(entry)
	}
	message.Chunks = entry.GetChunks()
	message.Extended = entry.Extended
	message.HardLinkId = entry.HardLinkId
	message.HardLinkCounter = entry.HardLinkCounter
	message.Content = entry.Content
	message.RemoteEntry = entry.Remote
	message.Quota = entry.Quota
	message.WormEnforcedAtTsNs = entry.WORMEnforcedAtTsNs
	message.CommitTsNs = entry.CommitTsNs
}

func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) {
	fsEntry.Attr = PbToEntryAttribute(message.Attributes)
	fsEntry.Chunks = message.Chunks
	fsEntry.Extended = message.Extended
	fsEntry.HardLinkId = HardLinkId(message.HardLinkId)
	fsEntry.HardLinkCounter = message.HardLinkCounter
	fsEntry.Content = message.Content
	fsEntry.Remote = message.RemoteEntry
	fsEntry.Quota = message.Quota
	fsEntry.FileSize = FileSize(message)
	fsEntry.WORMEnforcedAtTsNs = message.WormEnforcedAtTsNs
	fsEntry.CommitTsNs = message.CommitTsNs
}

func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
	if entry == nil {
		return nil
	}
	dir, _ := entry.FullPath.DirAndName()
	return &filer_pb.FullEntry{
		Dir:   dir,
		Entry: entry.ToProtoEntry(),
	}
}

func (entry *Entry) GetChunks() []*filer_pb.FileChunk {
	return entry.Chunks
}

func FromPbEntry(dir string, entry *filer_pb.Entry) *Entry {
	t := &Entry{}
	t.FullPath = util.NewFullPath(dir, entry.Name)
	FromPbEntryToExistingEntry(entry, t)
	return t
}

func maxUint64(x, y uint64) uint64 {
	if x > y {
		return x
	}
	return y
}

func (entry *Entry) IsExpireS3Enabled() (exist bool) {
	if entry.Extended != nil {
		_, exist = entry.Extended[s3_constants.SeaweedFSExpiresS3]
	}
	return exist
}

func (entry *Entry) isS3Entry() bool {
	if entry.Extended == nil {
		return false
	}

	if _, found := entry.Extended[s3_constants.ExtAmzOwnerKey]; found {
		return true
	}

	if _, found := entry.Extended[s3_constants.ExtETagKey]; found {
		return true
	}

	for key := range entry.Extended {
		if strings.HasPrefix(key, s3_constants.ExtAmzPrefix) {
			return true
		}
	}
	return false
}

// ApplyStorageTtl stamps the TTL the entry's data was placed with. Remote
// entries never expire locally; the remote storage owns their lifecycle.
func (entry *Entry) ApplyStorageTtl(ttlSec int32) {
	if entry.Remote != nil {
		ttlSec = 0
	}
	entry.TtlSec = ttlSec
	entry.ApplyS3ExpiryMetadata()
}

func (entry *Entry) ApplyS3ExpiryMetadata() {
	if entry.TtlSec == 0 {
		return
	}

	if _, found := entry.Extended[s3_constants.SeaweedFSExpiresS3]; found {
		return
	}

	// The s3 expiry path skips versioned entries, so stamping one would drop its
	// expiry entirely instead of moving it onto mtime.
	if entry.IsS3Versioning() {
		return
	}

	if !entry.isS3Entry() {
		return
	}

	entry.Extended[s3_constants.SeaweedFSExpiresS3] = []byte("true")

	return
}

func (entry *Entry) IsS3Versioning() (exist bool) {
	if entry.Extended != nil {
		_, exist = entry.Extended[s3_constants.ExtVersionIdKey]
	}
	return exist
}

func (entry *Entry) GetS3ExpireTime() (expireTime time.Time) {
	if entry.Mtime.IsZero() {
		expireTime = entry.Crtime
	} else {
		expireTime = entry.Mtime
	}
	return expireTime.Add(time.Duration(entry.TtlSec) * time.Second)
}