SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
//go:build !freebsd
package mount
import (
"context"
"fmt"
"os"
"os/exec"
"runtime"
"strings"
"sync"
"testing"
"time"
"github.com/seaweedfs/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
// A subprocess bounds the deliberately deadlocked negative without leaving
// blocked goroutines/cache cleanup behind in the rest of the package suite.
func TestAttributeSetAttrReadLockOrder(t *testing.T) {
if os.Getenv("MOUNT_ATTR_READ_LOCK_CHILD") == "1" {
attributeReadLockScenario(t)
return
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestAttributeSetAttrReadLockOrder$", "-test.count=1", "-test.timeout=12s")
cmd.Env = append(os.Environ(), "MOUNT_ATTR_READ_LOCK_CHILD=1")
output, err := cmd.CombinedOutput()
if ctx.Err() != nil {
t.Fatalf("lock-order control failed to finish its bounded child: %v\n%s", ctx.Err(), output)
}
if err != nil {
t.Fatalf("concurrent SetAttr/Read did not complete: %v\n%s", err, output)
}
}
func attributeReadLockScenario(t *testing.T) {
w := newInvalidateTestWFS(t)
w.atimeMap = make(map[uint64]time.Time)
data := []byte("acknowledged payload")
current := attributeAppendEntry(0)
current.Content = data
current.Attributes.FileSize = uint64(len(data))
fake := &attributeCASFiler{entry: proto.Clone(current).(*filer_pb.Entry)}
committed, resume := make(chan struct{}), make(chan struct{})
var resumeOnce sync.Once
t.Cleanup(func() { resumeOnce.Do(func() { close(resume) }) })
fake.afterCommit = func() { close(committed); <-resume }
startFakeFiler(t, w, fake)
path := util.FullPath("/dir/file")
w.inodeToPath.Lookup(path, 1, false, false, 42, true)
local := proto.Clone(current).(*filer_pb.Entry)
w.mapPbIdFromFilerToLocal(local)
fh := newFileHandle(w, 1, 42, local)
fh.RememberPath(path)
w.fhMap.inode2fh[42], w.fhMap.fh2inode[1] = fh, 42
t.Cleanup(fh.dirtyPages.Destroy)
attrDone := make(chan fuse.Status, 1)
go func() {
in := &fuse.SetAttrIn{}
in.NodeId = 42
in.Valid = fuse.FATTR_MODE
in.Mode = 0600
attrDone <- w.SetAttr(nil, in, &fuse.AttrOut{})
}()
select {
case <-committed:
case <-time.After(5 * time.Second):
t.Fatal("SetAttr did not reach owner commit hook")
}
// Model the background WRITE which makes retention necessary after the owner
// has committed. No unprotected flag write overlaps a running handler.
active := w.fhLockTable.AcquireLock("test background write", fh.fh, util.ExclusiveLock)
fh.dirtyMetadata = true
w.fhLockTable.ReleaseLock(fh.fh, active)
// Hold a reader on entryLock until retention is demonstrably waiting for its
// writer lock. The old path has no active lock; the fix already owns it.
fh.entryLock.RLock()
resumeOnce.Do(func() { close(resume) })
waitAttributeReadFrame(t, "(*WFS).retainAcknowledgedAttributes", "sync.(*RWMutex).Lock")
readDone := make(chan fuse.Status, 1)
go func() {
buf := make([]byte, len(data))
result, status := w.Read(nil, &fuse.ReadIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: 1, Size: uint32(len(data))}, buf)
if status == fuse.OK {
got, code := result.Bytes(buf)
if code != fuse.OK || string(got) != string(data) {
status = fuse.EIO
}
result.Done()
}
readDone <- status
}()
// Prove the real Read has arrived before opening the gate: old code waits
// on entryLock while holding the table entry, fixed code waits on the table.
waitAttributeReadFrame(t, "(*WFS).Read", "(*FileHandle).readFromChunksWithContext", "AcquireLock")
fh.entryLock.RUnlock()
timer := time.NewTimer(2 * time.Second)
defer timer.Stop()
for _, done := range []chan fuse.Status{attrDone, readDone} {
select {
case status := <-done:
if status != fuse.OK {
t.Fatalf("handler returned %v", status)
}
case <-timer.C:
fmt.Fprintln(os.Stderr, "LOCK_ORDER_CYCLE: owner-committed SetAttr retention and same-handle Read cannot finish")
os.Exit(23)
}
}
fh.entry.RLock()
mode := fh.entry.Attributes.FileMode
fh.entry.RUnlock()
if mode&0777 != 0600 {
t.Fatalf("acknowledged chmod was not retained: %o", mode)
}
}
// The attribute snapshot must queue for the handle lock before it takes
// entryLock. Otherwise a Read holding the shared lock waits on entryLock, the
// snapshot waits behind a queued exclusive (an invalidation or Write), and
// that waits on the Read.
func TestAttributeSnapshotReadLockOrder(t *testing.T) {
w := newInvalidateTestWFS(t)
entry := attributeAppendEntry(0)
fh := newFileHandle(w, 1, 42, entry)
t.Cleanup(fh.dirtyPages.Destroy)
read := w.fhLockTable.AcquireLock("test read", fh.fh, util.SharedLock)
exclusiveDone := make(chan struct{})
go func() {
lock := w.fhLockTable.AcquireLock("test invalidate", fh.fh, util.ExclusiveLock)
w.fhLockTable.ReleaseLock(fh.fh, lock)
close(exclusiveDone)
}()
waitAttributeReadFrame(t, "TestAttributeSnapshotReadLockOrder.func1", "sync.(*Cond).Wait")
snapshotDone := make(chan bool, 1)
go func() {
_, ok := w.attributeEntrySnapshot(fh, entry)
snapshotDone <- ok
}()
waitAttributeReadFrame(t, "(*WFS).attributeEntrySnapshot", "sync.(*Cond).Wait")
readerCanProceed := fh.entryLock.TryRLock()
if readerCanProceed {
fh.entryLock.RUnlock()
}
w.fhLockTable.ReleaseLock(fh.fh, read)
<-exclusiveDone
if ok := <-snapshotDone; !ok {
t.Fatal("clean handle snapshot was refused")
}
if !readerCanProceed {
t.Fatal("snapshot holds entryLock while queued for the handle lock behind an exclusive waiter")
}
}
// Stack frames are arrival/blocked witnesses, not an elapsed-time performance
// assertion. Channel/lock gates define the order; timeout only refuses a broken
// test arrangement. No sleep is used to make either handler win a race.
func waitAttributeReadFrame(t *testing.T, caller string, blocked ...string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
for _, stack := range strings.Split(string(buf[:n]), "\n\n") {
if !strings.Contains(stack, caller) {
continue
}
for _, frame := range blocked {
if strings.Contains(stack, frame) {
return
}
}
}
runtime.Gosched()
}
t.Fatalf("did not observe %s blocked at %v", caller, blocked)
}
// FATTR_SIZE bypasses the attribute publication fence but must not reverse
// the handle locks while an acknowledged chmod is retaining its local delta.
func TestAttributeTruncateRetainLockOrder(t *testing.T) {
if os.Getenv("MOUNT_ATTR_TRUNCATE_LOCK_CHILD") == "1" {
attributeTruncateLockScenario(t)
return
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestAttributeTruncateRetainLockOrder$", "-test.count=1", "-test.timeout=12s")
cmd.Env = append(os.Environ(), "MOUNT_ATTR_TRUNCATE_LOCK_CHILD=1")
output, err := cmd.CombinedOutput()
if ctx.Err() != nil {
t.Fatalf("truncate lock control exceeded bounded child: %v\n%s", ctx.Err(), output)
}
if err != nil {
t.Fatalf("concurrent truncate/acked chmod did not finish: %v\n%s", err, output)
}
}
func attributeTruncateLockScenario(t *testing.T) {
w := newInvalidateTestWFS(t)
w.atimeMap = make(map[uint64]time.Time)
current := attributeAppendEntry(3)
fake := &attributeCASFiler{entry: proto.Clone(current).(*filer_pb.Entry)}
committed, resume := make(chan struct{}), make(chan struct{})
var once sync.Once
t.Cleanup(func() { once.Do(func() { close(resume) }) })
fake.afterCommit = func() { close(committed); <-resume }
startFakeFiler(t, w, fake)
path := util.FullPath("/dir/file")
w.inodeToPath.Lookup(path, 1, false, false, 42, true)
local := proto.Clone(current).(*filer_pb.Entry)
w.mapPbIdFromFilerToLocal(local)
fh := newFileHandle(w, 1, 42, local)
fh.RememberPath(path)
w.fhMap.inode2fh[42], w.fhMap.fh2inode[1] = fh, 42
t.Cleanup(fh.dirtyPages.Destroy)
chmodDone := make(chan fuse.Status, 1)
go func() {
in := &fuse.SetAttrIn{}
in.NodeId = 42
in.Valid = fuse.FATTR_MODE
in.Mode = 0600
chmodDone <- w.SetAttr(nil, in, &fuse.AttrOut{})
}()
select {
case <-committed:
case <-time.After(5 * time.Second):
t.Fatal("chmod did not reach owner commit hook")
}
active := w.fhLockTable.AcquireLock("test pending content", fh.fh, util.ExclusiveLock)
fh.dirtyMetadata = true
w.fhLockTable.ReleaseLock(fh.fh, active)
fh.entryLock.RLock()
truncateDone := make(chan fuse.Status, 1)
go func() {
in := &fuse.SetAttrIn{}
in.NodeId = 42
in.Valid = fuse.FATTR_SIZE
in.Size = 256
truncateDone <- w.SetAttr(nil, in, &fuse.AttrOut{})
}()
// Truncate queued its entryLock writer before the chmod response is released.
waitAttributeLockPair(t, [][2]string{{"(*WFS).attributeEntrySnapshot", "sync.(*RWMutex).Lock"}})
once.Do(func() { close(resume) })
// Rejected head: retention owns active-exclusive and waits for entryLock.
// Corrected head: truncate owns active-shared and installation waits for it.
waitAttributeLockPair(t, [][2]string{{"(*WFS).retainAcknowledgedAttributes", "sync.(*RWMutex).Lock"}, {"(*FileHandle).installAckedEntry", "AcquireLock"}})
fh.entryLock.RUnlock()
timer := time.NewTimer(2 * time.Second)
defer timer.Stop()
for _, done := range []chan fuse.Status{chmodDone, truncateDone} {
select {
case status := <-done:
if status != fuse.OK {
t.Fatalf("handler returned %v", status)
}
case <-timer.C:
fmt.Fprintln(os.Stderr, "TRUNCATE_RETENTION_CYCLE: FATTR_SIZE snapshot and acknowledged chmod retention cannot finish")
os.Exit(24)
}
}
fh.entry.RLock()
defer fh.entry.RUnlock()
e := fh.entry.Entry
if e.Attributes.FileMode&0777 != 0600 || e.Attributes.FileSize != 256 || len(e.Chunks) != 1 || !proto.Equal(e.Chunks[0], current.Chunks[0]) {
t.Fatalf("lost acknowledged chmod or pending truncate: %v", e)
}
}
func waitAttributeLockPair(t *testing.T, pairs [][2]string) {
t.Helper()
end := time.Now().Add(5 * time.Second)
for time.Now().Before(end) {
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
for _, stack := range strings.Split(string(buf[:n]), "\n\n") {
header, _, _ := strings.Cut(stack, "\n")
if !strings.Contains(header, "[sync.") && !strings.Contains(header, "[semacquire") {
continue
}
for _, pair := range pairs {
if strings.Contains(stack, pair[0]) && strings.Contains(stack, pair[1]) {
return
}
}
}
runtime.Gosched()
}
t.Fatalf("did not observe actual blocked lock pair %v", pairs)
}