SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package filer
import (
"os"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type Attr struct {
Mtime time.Time // time of last modification
Crtime time.Time // time of creation (OS X only)
Ctime time.Time // time of last inode change
Atime time.Time // time of last access
Mode os.FileMode // file mode
Uid uint32 // owner uid
Gid uint32 // group gid
Mime string // mime type
TtlSec int32 // ttl in seconds
UserName string
GroupNames []string
SymlinkTarget string
Md5 []byte
FileSize uint64
Rdev uint32
Inode uint64
}
func (attr Attr) IsDirectory() bool {
return attr.Mode&os.ModeDir > 0
}
type Entry struct {
util.FullPath
Attr
Extended map[string][]byte
// the following is for files
Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"`
HardLinkId HardLinkId
HardLinkCounter int32
Content []byte
Remote *filer_pb.RemoteEntry
Quota int64
WORMEnforcedAtTsNs int64
// CommitTsNs is the meta log event timestamp of the mutation that
// produced this version. Zero for pre-field versions; they order by mtime.
CommitTsNs int64
}
func (entry *Entry) Size() uint64 {
return maxUint64(maxUint64(TotalSize(entry.GetChunks()), entry.FileSize), uint64(len(entry.Content)))
}
func (entry *Entry) Timestamp() time.Time {
if entry.IsDirectory() {
return entry.Crtime
} else {
return entry.Mtime
}
}
func (entry *Entry) ShallowClone() *Entry {
if entry == nil {
return nil
}
newEntry := &Entry{}
newEntry.FullPath = entry.FullPath
newEntry.Attr = entry.Attr
newEntry.Chunks = entry.Chunks
newEntry.Extended = entry.Extended
newEntry.HardLinkId = entry.HardLinkId
newEntry.HardLinkCounter = entry.HardLinkCounter
newEntry.Content = entry.Content
newEntry.Remote = entry.Remote
newEntry.Quota = entry.Quota
newEntry.WORMEnforcedAtTsNs = entry.WORMEnforcedAtTsNs
newEntry.CommitTsNs = entry.CommitTsNs
return newEntry
}
func (entry *Entry) ToProtoEntry() *filer_pb.Entry {
if entry == nil {
return nil
}
message := &filer_pb.Entry{}
message.Name = entry.FullPath.Name()
entry.ToExistingProtoEntry(message)
return message
}
func (entry *Entry) ToExistingProtoEntry(message *filer_pb.Entry) {
if entry == nil {
return
}
message.IsDirectory = entry.IsDirectory()
// Reuse pre-allocated attributes if available, otherwise allocate
if message.Attributes != nil {
EntryAttributeToExistingPb(entry, message.Attributes)
} else {
message.Attributes = EntryAttributeToPb(entry)
}
message.Chunks = entry.GetChunks()
message.Extended = entry.Extended
message.HardLinkId = entry.HardLinkId
message.HardLinkCounter = entry.HardLinkCounter
message.Content = entry.Content
message.RemoteEntry = entry.Remote
message.Quota = entry.Quota
message.WormEnforcedAtTsNs = entry.WORMEnforcedAtTsNs
message.CommitTsNs = entry.CommitTsNs
}
func FromPbEntryToExistingEntry(message *filer_pb.Entry, fsEntry *Entry) {
fsEntry.Attr = PbToEntryAttribute(message.Attributes)
fsEntry.Chunks = message.Chunks
fsEntry.Extended = message.Extended
fsEntry.HardLinkId = HardLinkId(message.HardLinkId)
fsEntry.HardLinkCounter = message.HardLinkCounter
fsEntry.Content = message.Content
fsEntry.Remote = message.RemoteEntry
fsEntry.Quota = message.Quota
fsEntry.FileSize = FileSize(message)
fsEntry.WORMEnforcedAtTsNs = message.WormEnforcedAtTsNs
fsEntry.CommitTsNs = message.CommitTsNs
}
func (entry *Entry) ToProtoFullEntry() *filer_pb.FullEntry {
if entry == nil {
return nil
}
dir, _ := entry.FullPath.DirAndName()
return &filer_pb.FullEntry{
Dir: dir,
Entry: entry.ToProtoEntry(),
}
}
func (entry *Entry) GetChunks() []*filer_pb.FileChunk {
return entry.Chunks
}
func FromPbEntry(dir string, entry *filer_pb.Entry) *Entry {
t := &Entry{}
t.FullPath = util.NewFullPath(dir, entry.Name)
FromPbEntryToExistingEntry(entry, t)
return t
}
func maxUint64(x, y uint64) uint64 {
if x > y {
return x
}
return y
}
func (entry *Entry) IsExpireS3Enabled() (exist bool) {
if entry.Extended != nil {
_, exist = entry.Extended[s3_constants.SeaweedFSExpiresS3]
}
return exist
}
func (entry *Entry) isS3Entry() bool {
if entry.Extended == nil {
return false
}
if _, found := entry.Extended[s3_constants.ExtAmzOwnerKey]; found {
return true
}
if _, found := entry.Extended[s3_constants.ExtETagKey]; found {
return true
}
for key := range entry.Extended {
if strings.HasPrefix(key, s3_constants.ExtAmzPrefix) {
return true
}
}
return false
}
// ApplyStorageTtl stamps the TTL the entry's data was placed with. Remote
// entries never expire locally; the remote storage owns their lifecycle.
func (entry *Entry) ApplyStorageTtl(ttlSec int32) {
if entry.Remote != nil {
ttlSec = 0
}
entry.TtlSec = ttlSec
entry.ApplyS3ExpiryMetadata()
}
func (entry *Entry) ApplyS3ExpiryMetadata() {
if entry.TtlSec == 0 {
return
}
if _, found := entry.Extended[s3_constants.SeaweedFSExpiresS3]; found {
return
}
// The s3 expiry path skips versioned entries, so stamping one would drop its
// expiry entirely instead of moving it onto mtime.
if entry.IsS3Versioning() {
return
}
if !entry.isS3Entry() {
return
}
entry.Extended[s3_constants.SeaweedFSExpiresS3] = []byte("true")
return
}
func (entry *Entry) IsS3Versioning() (exist bool) {
if entry.Extended != nil {
_, exist = entry.Extended[s3_constants.ExtVersionIdKey]
}
return exist
}
func (entry *Entry) GetS3ExpireTime() (expireTime time.Time) {
if entry.Mtime.IsZero() {
expireTime = entry.Crtime
} else {
expireTime = entry.Mtime
}
return expireTime.Add(time.Duration(entry.TtlSec) * time.Second)
}