Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

mount: fold an inode's single link into its entry (#10818)

Chris Lubb223967bd7c
diff --git a/weed/mount/inode_to_path.go b/weed/mount/inode_to_path.go
index 3a65958353..4b669535fb 100644
--- a/weed/mount/inode_to_path.go
+++ b/weed/mount/inode_to_path.go
@@ -22,10 +22,12 @@ type InodeToPath struct {
// InodeEntry exists per inode the kernel references. Directory cache state is
// kept out in dirStates so a file entry stays in the 32-byte size class — the
-// dominant cost on a mount with millions of files.
+// dominant cost on a mount with millions of files. A hard link's extra paths
+// hang off a pointer for the same reason.
type InodeEntry struct {
- paths []util.FullPath
- nlookup uint64
+ path util.FullPath
+ nlookup uint64
+ extraPaths *[]util.FullPath
}
type dirState struct {
@@ -43,25 +45,80 @@ func (d *dirState) resetCacheState() {
d.cachedExpiresTime = time.Time{}
}
+// appendPaths appends every path the inode is reachable by, primary first.
+func (ie *InodeEntry) appendPaths(dst []util.FullPath) []util.FullPath {
+ if ie.path == "" {
+ return dst
+ }
+ dst = append(dst, ie.path)
+ if ie.extraPaths != nil {
+ dst = append(dst, *ie.extraPaths...)
+ }
+ return dst
+}
+
+func (ie *InodeEntry) addPath(p util.FullPath) {
+ if ie.path == "" {
+ ie.path = p
+ return
+ }
+ if ie.extraPaths == nil {
+ ie.extraPaths = &[]util.FullPath{p}
+ return
+ }
+ *ie.extraPaths = append(*ie.extraPaths, p)
+}
+
+func (ie *InodeEntry) replacePath(from, to util.FullPath) {
+ if ie.path == from {
+ ie.path = to
+ }
+ if ie.extraPaths == nil {
+ return
+ }
+ for i, p := range *ie.extraPaths {
+ if p == from {
+ (*ie.extraPaths)[i] = to
+ }
+ }
+}
+
+func (ie *InodeEntry) setExtraPaths(extra []util.FullPath) {
+ if len(extra) == 0 {
+ ie.extraPaths = nil
+ return
+ }
+ ie.extraPaths = &extra
+}
+
+// removeOnePath promotes an extra path when the primary is the one going away,
+// so an entry that still has a path always has a primary one.
func (ie *InodeEntry) removeOnePath(p util.FullPath) bool {
- if len(ie.paths) == 0 {
+ if ie.path == "" {
return false
}
- idx := -1
- for i, x := range ie.paths {
- if x == p {
- idx = i
- break
+ if ie.path == p {
+ if ie.extraPaths == nil {
+ ie.path = ""
+ return true
}
+ extra := *ie.extraPaths
+ ie.path = extra[0]
+ ie.setExtraPaths(extra[1:])
+ return true
}
- if idx < 0 {
+ if ie.extraPaths == nil {
return false
}
- for x := idx; x < len(ie.paths)-1; x++ {
- ie.paths[x] = ie.paths[x+1]
+ extra := *ie.extraPaths
+ for i, x := range extra {
+ if x != p {
+ continue
+ }
+ ie.setExtraPaths(append(extra[:i], extra[i+1:]...))
+ return true
}
- ie.paths = ie.paths[0 : len(ie.paths)-1]
- return true
+ return false
}
func NewInodeToPath(root util.FullPath, ttlSec int) *InodeToPath {
@@ -72,7 +129,7 @@ func NewInodeToPath(root util.FullPath, ttlSec int) *InodeToPath {
cacheMetaTtlSec: time.Second * time.Duration(ttlSec),
}
t.inode2path[1] = &InodeEntry{
- paths: []util.FullPath{root},
+ path: root,
nlookup: 1,
}
t.dirStates[1] = &dirState{lastAccess: time.Now()}
@@ -122,7 +179,7 @@ func (i *InodeToPath) Lookup(path util.FullPath, unixTime int64, isDirectory boo
nlookup = 1
}
i.inode2path[inode] = &InodeEntry{
- paths: []util.FullPath{path},
+ path: path,
nlookup: nlookup,
}
if isDirectory {
@@ -193,10 +250,10 @@ func (i *InodeToPath) GetPath(inode uint64) (util.FullPath, fuse.Status) {
i.RLock()
defer i.RUnlock()
path, found := i.inode2path[inode]
- if !found || len(path.paths) == 0 {
+ if !found || path.path == "" {
return "", fuse.ENOENT
}
- return path.paths[0], fuse.OK
+ return path.path, fuse.OK
}
// GetAllPaths returns a copy of all paths associated with an inode. For a
@@ -206,12 +263,10 @@ func (i *InodeToPath) GetAllPaths(inode uint64) []util.FullPath {
i.RLock()
defer i.RUnlock()
ie, found := i.inode2path[inode]
- if !found || len(ie.paths) == 0 {
+ if !found {
return nil
}
- out := make([]util.FullPath, len(ie.paths))
- copy(out, ie.paths)
- return out
+ return ie.appendPaths(nil)
}
func (i *InodeToPath) HasPath(path util.FullPath) bool {
@@ -422,7 +477,7 @@ func (i *InodeToPath) CollectEvictableDirs(now time.Time, idle time.Duration) []
}
d.resetCacheState()
if entry, ok := i.inode2path[inode]; ok {
- dirs = append(dirs, entry.paths...)
+ dirs = entry.appendPaths(dirs)
}
}
return dirs
@@ -435,11 +490,11 @@ func (i *InodeToPath) AddPath(inode uint64, path util.FullPath) {
ie, found := i.inode2path[inode]
if found {
- ie.paths = append(ie.paths, path)
+ ie.addPath(path)
ie.nlookup++
} else {
i.inode2path[inode] = &InodeEntry{
- paths: []util.FullPath{path},
+ path: path,
nlookup: 1,
}
}
@@ -483,11 +538,7 @@ func (i *InodeToPath) MovePath(sourcePath, targetPath util.FullPath) (sourceInod
return
}
if entry, entryFound := i.inode2path[sourceInode]; entryFound {
- for i, p := range entry.paths {
- if p == sourcePath {
- entry.paths[i] = targetPath
- }
- }
+ entry.replacePath(sourcePath, targetPath)
if d := i.dirStates[sourceInode]; d != nil {
d.resetCacheState()
}
@@ -509,27 +560,30 @@ func (i *InodeToPath) Forget(inode, nlookup uint64, onRelease func(inode uint64)
path, found := i.inode2path[inode]
if found {
if nlookup > path.nlookup {
- glog.Errorf("kernel forget over-decrement: inode %d paths %v current %d forget %d", inode, path.paths, path.nlookup, nlookup)
+ glog.Errorf("kernel forget over-decrement: inode %d path %v current %d forget %d", inode, path.path, path.nlookup, nlookup)
path.nlookup = 0
} else {
path.nlookup -= nlookup
}
- glog.V(4).Infof("kernel forget: inode %d paths %v nlookup %d", inode, path.paths, path.nlookup)
+ glog.V(4).Infof("kernel forget: inode %d path %v nlookup %d", inode, path.path, path.nlookup)
if path.nlookup == 0 {
if onRelease != nil {
onRelease(inode)
}
if _, isDir := i.dirStates[inode]; isDir && onForgetDir != nil {
- dirPaths = append([]util.FullPath(nil), path.paths...)
+ dirPaths = path.appendPaths(nil)
callOnForgetDir = true
}
- for _, p := range path.paths {
- delete(i.path2inode, p)
+ delete(i.path2inode, path.path)
+ if path.extraPaths != nil {
+ for _, p := range *path.extraPaths {
+ delete(i.path2inode, p)
+ }
}
delete(i.inode2path, inode)
delete(i.dirStates, inode)
} else {
- glog.V(4).Infof("kernel forget but nlookup not zero: inode %d paths %v nlookup %d", inode, path.paths, path.nlookup)
+ glog.V(4).Infof("kernel forget but nlookup not zero: inode %d path %v nlookup %d", inode, path.path, path.nlookup)
}
} else {
glog.Warningf("kernel forget but inode not found: inode %d", inode)
diff --git a/weed/mount/inode_to_path_test.go b/weed/mount/inode_to_path_test.go
index e76dd45834..378bfd02f2 100644
--- a/weed/mount/inode_to_path_test.go
+++ b/weed/mount/inode_to_path_test.go
@@ -3,10 +3,29 @@ package mount
import (
"testing"
"time"
+ "unsafe"
"github.com/seaweedfs/seaweedfs/weed/util"
)
+// A mount with millions of files is mostly these, so guard the size class.
+func TestInodeEntryStaysInSizeClass(t *testing.T) {
+ if unsafe.Sizeof(uintptr(0)) != 8 {
+ t.Skip("size classes are 64-bit specific")
+ }
+ if got := unsafe.Sizeof(InodeEntry{}); got != 32 {
+ t.Errorf("sizeof(InodeEntry) = %d, want 32", got)
+ }
+}
+
+func inodeEntryWith(paths ...util.FullPath) InodeEntry {
+ var ie InodeEntry
+ for _, p := range paths {
+ ie.addPath(p)
+ }
+ return ie
+}
+
func TestInodeEntry_removeOnePath(t *testing.T) {
tests := []struct {
name string
@@ -16,10 +35,8 @@ func TestInodeEntry_removeOnePath(t *testing.T) {
count int
}{
{
- name: "actual case",
- entry: InodeEntry{
- paths: []util.FullPath{"/pjd/nx", "/pjd/n0"},
- },
+ name: "actual case",
+ entry: inodeEntryWith("/pjd/nx", "/pjd/n0"),
p: "/pjd/nx",
want: true,
count: 1,
@@ -32,46 +49,36 @@ func TestInodeEntry_removeOnePath(t *testing.T) {
count: 0,
},
{
- name: "single",
- entry: InodeEntry{
- paths: []util.FullPath{"/x"},
- },
+ name: "single",
+ entry: inodeEntryWith("/x"),
p: "/x",
want: true,
count: 0,
},
{
- name: "first",
- entry: InodeEntry{
- paths: []util.FullPath{"/x", "/y", "/z"},
- },
+ name: "first",
+ entry: inodeEntryWith("/x", "/y", "/z"),
p: "/x",
want: true,
count: 2,
},
{
- name: "middle",
- entry: InodeEntry{
- paths: []util.FullPath{"/x", "/y", "/z"},
- },
+ name: "middle",
+ entry: inodeEntryWith("/x", "/y", "/z"),
p: "/y",
want: true,
count: 2,
},
{
- name: "last",
- entry: InodeEntry{
- paths: []util.FullPath{"/x", "/y", "/z"},
- },
+ name: "last",
+ entry: inodeEntryWith("/x", "/y", "/z"),
p: "/z",
want: true,
count: 2,
},
{
- name: "not found",
- entry: InodeEntry{
- paths: []util.FullPath{"/x", "/y", "/z"},
- },
+ name: "not found",
+ entry: inodeEntryWith("/x", "/y", "/z"),
p: "/t",
want: false,
count: 3,
@@ -82,10 +89,11 @@ func TestInodeEntry_removeOnePath(t *testing.T) {
if got := tt.entry.removeOnePath(tt.p); got != tt.want {
t.Errorf("removeOnePath() = %v, want %v", got, tt.want)
}
- if tt.count != len(tt.entry.paths) {
- t.Errorf("removeOnePath path count = %v, want %v", len(tt.entry.paths), tt.count)
+ left := tt.entry.appendPaths(nil)
+ if tt.count != len(left) {
+ t.Errorf("removeOnePath path count = %v, want %v", len(left), tt.count)
}
- for i, p := range tt.entry.paths {
+ for i, p := range left {
if p == tt.p {
t.Errorf("removeOnePath found path still exists at %v, %v", i, p)
}