SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package distributed_lock
import (
"bufio"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/seaweedfs/seaweedfs/test/testutil"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
const (
distributedLockTestRegion = "us-east-1"
distributedLockTestAccessKey = "some_access_key1"
distributedLockTestSecretKey = "some_secret_key1"
distributedLockTestGroup = "distributed-lock-it"
)
type distributedLockCluster struct {
t testing.TB
baseDir string
configDir string
logsDir string
keepLogs bool
weedBinary string
filerGroup string
s3Config string
masterPort int
masterGrpcPort int
volumePort int
volumeGrpcPort int
filerPorts []int
filerGrpcPorts []int
s3Ports []int
s3GrpcPorts []int
masterCmd *exec.Cmd
volumeCmd *exec.Cmd
filerCmds []*exec.Cmd
s3Cmds []*exec.Cmd
logFiles []*os.File
cleanupOnce sync.Once
}
type s3IdentityConfig struct {
Identities []s3Identity `json:"identities"`
}
type s3Identity struct {
Name string `json:"name"`
Credentials []s3Credential `json:"credentials,omitempty"`
Actions []string `json:"actions"`
}
type s3Credential struct {
AccessKey string `json:"accessKey"`
SecretKey string `json:"secretKey"`
}
func startDistributedLockCluster(t *testing.T) *distributedLockCluster {
t.Helper()
weedBinary, err := framework.FindOrBuildWeedBinary()
require.NoError(t, err, "resolve weed binary")
baseDir, err := os.MkdirTemp("", "seaweedfs_s3_distributed_lock_")
require.NoError(t, err, "create temp directory")
cluster := &distributedLockCluster{
t: t,
baseDir: baseDir,
configDir: filepath.Join(baseDir, "config"),
logsDir: filepath.Join(baseDir, "logs"),
keepLogs: os.Getenv("S3_DISTRIBUTED_LOCK_KEEP_LOGS") == "1",
weedBinary: weedBinary,
filerGroup: distributedLockTestGroup,
filerCmds: make([]*exec.Cmd, 0, 2),
s3Cmds: make([]*exec.Cmd, 0, 2),
}
t.Cleanup(cluster.Stop)
dirs := []string{
cluster.configDir,
cluster.logsDir,
filepath.Join(baseDir, "master"),
filepath.Join(baseDir, "volume"),
filepath.Join(baseDir, "filer0"),
filepath.Join(baseDir, "filer1"),
}
for _, dir := range dirs {
require.NoError(t, os.MkdirAll(dir, 0o755), "create %s", dir)
}
ports, err := testutil.AllocatePorts(12)
require.NoError(t, err, "allocate ports")
cluster.masterPort = ports[0]
cluster.masterGrpcPort = ports[1]
cluster.volumePort = ports[2]
cluster.volumeGrpcPort = ports[3]
cluster.filerPorts = []int{ports[4], ports[6]}
cluster.filerGrpcPorts = []int{ports[5], ports[7]}
cluster.s3Ports = []int{ports[8], ports[10]}
cluster.s3GrpcPorts = []int{ports[9], ports[11]}
require.NoError(t, cluster.writeSecurityConfig(), "write security config")
require.NoError(t, cluster.writeS3Config(), "write s3 config")
require.NoError(t, cluster.startMaster(), "start master")
require.NoError(t, cluster.waitForHTTP("http://"+cluster.masterHTTPAddress()+"/dir/status", 30*time.Second), "wait for master\n%s", cluster.tailLog("master.log"))
require.NoError(t, cluster.startVolume(), "start volume")
require.NoError(t, cluster.waitForHTTP("http://"+cluster.volumeHTTPAddress()+"/status", 30*time.Second), "wait for volume\n%s", cluster.tailLog("volume.log"))
require.NoError(t, cluster.waitForTCP(cluster.volumeGRPCAddress(), 30*time.Second), "wait for volume grpc\n%s", cluster.tailLog("volume.log"))
for i := 0; i < 2; i++ {
require.NoError(t, cluster.startFiler(i), "start filer %d", i)
require.NoError(t, cluster.waitForTCP(cluster.filerGRPCAddress(i), 30*time.Second), "wait for filer %d grpc\n%s", i, cluster.tailLog(fmt.Sprintf("filer%d.log", i)))
}
require.NoError(t, cluster.waitForFilerCount(2, 30*time.Second), "wait for filer group registration")
require.NoError(t, cluster.waitForLockRingConverged(30*time.Second), "wait for lock ring convergence")
for i := 0; i < 2; i++ {
require.NoError(t, cluster.startS3(i), "start s3 %d", i)
client := cluster.newS3Client(t, cluster.s3Endpoint(i))
require.NoError(t, cluster.waitForS3Ready(client, 30*time.Second), "wait for s3 %d\n%s", i, cluster.tailLog(fmt.Sprintf("s3-%d.log", i)))
}
return cluster
}
func (c *distributedLockCluster) Stop() {
if c == nil {
return
}
c.cleanupOnce.Do(func() {
for i := len(c.s3Cmds) - 1; i >= 0; i-- {
stopProcess(c.s3Cmds[i])
}
for i := len(c.filerCmds) - 1; i >= 0; i-- {
stopProcess(c.filerCmds[i])
}
stopProcess(c.volumeCmd)
stopProcess(c.masterCmd)
for _, f := range c.logFiles {
_ = f.Close()
}
if !c.keepLogs && !c.t.Failed() {
_ = os.RemoveAll(c.baseDir)
} else if c.baseDir != "" {
c.t.Logf("distributed lock integration logs kept at %s", c.baseDir)
}
})
}
func (c *distributedLockCluster) masterHTTPAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterPort))
}
func (c *distributedLockCluster) masterGRPCAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.masterGrpcPort))
}
func (c *distributedLockCluster) volumeHTTPAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumePort))
}
func (c *distributedLockCluster) volumeGRPCAddress() string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.volumeGrpcPort))
}
func (c *distributedLockCluster) filerServerAddress(index int) pb.ServerAddress {
return pb.NewServerAddress("127.0.0.1", c.filerPorts[index], c.filerGrpcPorts[index])
}
func (c *distributedLockCluster) filerGRPCAddress(index int) string {
return net.JoinHostPort("127.0.0.1", strconv.Itoa(c.filerGrpcPorts[index]))
}
func (c *distributedLockCluster) s3Endpoint(index int) string {
return fmt.Sprintf("http://127.0.0.1:%d", c.s3Ports[index])
}
func (c *distributedLockCluster) startMaster() error {
logFile, err := c.openLog("master.log")
if err != nil {
return err
}
args := []string{
"-config_dir=" + c.configDir,
"master",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port=" + strconv.Itoa(c.masterPort),
"-port.grpc=" + strconv.Itoa(c.masterGrpcPort),
"-mdir=" + filepath.Join(c.baseDir, "master"),
"-peers=none",
"-volumeSizeLimitMB=32",
"-defaultReplication=000",
}
c.masterCmd = exec.Command(c.weedBinary, args...)
c.masterCmd.Dir = c.baseDir
c.masterCmd.Stdout = logFile
c.masterCmd.Stderr = logFile
return c.masterCmd.Start()
}
func (c *distributedLockCluster) startVolume() error {
logFile, err := c.openLog("volume.log")
if err != nil {
return err
}
masterAddress := string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
args := []string{
"-config_dir=" + c.configDir,
"volume",
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port=" + strconv.Itoa(c.volumePort),
"-port.grpc=" + strconv.Itoa(c.volumeGrpcPort),
"-dir=" + filepath.Join(c.baseDir, "volume"),
"-max=16",
"-master=" + masterAddress,
"-readMode=proxy",
}
c.volumeCmd = exec.Command(c.weedBinary, args...)
c.volumeCmd.Dir = c.baseDir
c.volumeCmd.Stdout = logFile
c.volumeCmd.Stderr = logFile
return c.volumeCmd.Start()
}
func (c *distributedLockCluster) startFiler(index int) error {
logFile, err := c.openLog(fmt.Sprintf("filer%d.log", index))
if err != nil {
return err
}
masterAddress := string(pb.NewServerAddress("127.0.0.1", c.masterPort, c.masterGrpcPort))
args := []string{
"-config_dir=" + c.configDir,
"filer",
"-master=" + masterAddress,
"-filerGroup=" + c.filerGroup,
"-ip=127.0.0.1",
"-ip.bind=127.0.0.1",
"-port=" + strconv.Itoa(c.filerPorts[index]),
"-port.grpc=" + strconv.Itoa(c.filerGrpcPorts[index]),
"-defaultStoreDir=" + filepath.Join(c.baseDir, fmt.Sprintf("filer%d", index)),
}
cmd := exec.Command(c.weedBinary, args...)
cmd.Dir = c.baseDir
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return err
}
c.filerCmds = append(c.filerCmds, cmd)
return nil
}
func (c *distributedLockCluster) startS3(index int) error {
logFile, err := c.openLog(fmt.Sprintf("s3-%d.log", index))
if err != nil {
return err
}
filers := []string{string(c.filerServerAddress(0)), string(c.filerServerAddress(1))}
if index%2 == 1 {
filers[0], filers[1] = filers[1], filers[0]
}
args := []string{
"-config_dir=" + c.configDir,
"s3",
"-ip.bind=127.0.0.1",
"-port=" + strconv.Itoa(c.s3Ports[index]),
"-port.grpc=" + strconv.Itoa(c.s3GrpcPorts[index]),
"-port.iceberg=0",
"-port.lance=0",
"-filer=" + strings.Join(filers, ","),
"-config=" + c.s3Config,
"-iam.readOnly=false",
}
cmd := exec.Command(c.weedBinary, args...)
cmd.Dir = c.baseDir
cmd.Stdout = logFile
cmd.Stderr = logFile
if err := cmd.Start(); err != nil {
return err
}
c.s3Cmds = append(c.s3Cmds, cmd)
return nil
}
func (c *distributedLockCluster) writeSecurityConfig() error {
return os.WriteFile(filepath.Join(c.configDir, "security.toml"), []byte("# generated for distributed lock integration tests\n"), 0o644)
}
func (c *distributedLockCluster) writeS3Config() error {
configPath := filepath.Join(c.configDir, "s3.json")
payload := s3IdentityConfig{
Identities: []s3Identity{
{
Name: "distributed-lock-admin",
Credentials: []s3Credential{
{
AccessKey: distributedLockTestAccessKey,
SecretKey: distributedLockTestSecretKey,
},
},
Actions: []string{"Admin", "Read", "List", "Tagging", "Write"},
},
},
}
data, err := json.MarshalIndent(payload, "", " ")
if err != nil {
return err
}
if err := os.WriteFile(configPath, data, 0o644); err != nil {
return err
}
c.s3Config = configPath
return nil
}
func (c *distributedLockCluster) newS3Client(t testing.TB, endpoint string) *s3.Client {
t.Helper()
cfg, err := config.LoadDefaultConfig(context.Background(),
config.WithRegion(distributedLockTestRegion),
config.WithRetryMaxAttempts(1),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
distributedLockTestAccessKey,
distributedLockTestSecretKey,
"",
)),
)
require.NoError(t, err, "load aws config")
return s3.NewFromConfig(cfg, func(o *s3.Options) {
o.BaseEndpoint = aws.String(endpoint)
o.UsePathStyle = true
})
}
func (c *distributedLockCluster) waitForS3Ready(client *s3.Client, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
_, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
cancel()
if err == nil {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("timed out waiting for s3 readiness")
}
func (c *distributedLockCluster) waitForFilerCount(expected int, timeout time.Duration) error {
conn, err := grpc.NewClient(c.masterGRPCAddress(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return err
}
defer conn.Close()
client := master_pb.NewSeaweedClient(conn)
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{
ClientType: "filer",
FilerGroup: c.filerGroup,
})
cancel()
if err == nil && len(resp.ClusterNodes) >= expected {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("timed out waiting for %d filers in group %q", expected, c.filerGroup)
}
// waitForLockRingConverged verifies that both filers have a consistent view of the
// lock ring by acquiring the same lock through each filer and checking mutual exclusion.
// This guards against the window between master seeing both filers and the filers
// actually receiving the LockRingUpdate broadcast (delayed by the stabilization timer).
//
// A single arbitrary key could false-pass: if the key's real primary is filer0 and
// filer0 has a stale ring (only sees itself), it still grants correctly because it IS
// the primary. To catch the stale ring we must also test a key whose real primary is
// filer1. So we generate one test key per primary filer using the same consistent-hash
// ring as production, and require mutual exclusion for all of them.
func (c *distributedLockCluster) waitForLockRingConverged(timeout time.Duration) error {
deadline := time.Now().Add(timeout)
owners := make([]pb.ServerAddress, 0, len(c.filerPorts))
for i := range c.filerPorts {
owners = append(owners, c.filerServerAddress(i))
}
ring := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
ring.SetServers(owners)
attempt := 0
for time.Now().Before(deadline) {
// Generate one unique test key per primary filer
testKeys := c.convergenceKeysPerPrimary(ring, owners, attempt)
attempt++
allConverged := true
for _, key := range testKeys {
converged, _ := c.checkLockMutualExclusion(key)
if !converged {
allConverged = false
break
}
}
if allConverged {
return nil
}
time.Sleep(500 * time.Millisecond)
}
return fmt.Errorf("lock ring did not converge: both filers independently grant the same lock")
}
// convergenceKeysPerPrimary returns one lock key per distinct primary filer,
// using the same consistent-hash ring as production routing.
func (c *distributedLockCluster) convergenceKeysPerPrimary(ring *lock_manager.HashRing, owners []pb.ServerAddress, attempt int) []string {
keysByOwner := make(map[pb.ServerAddress]string, len(owners))
for i := 0; len(keysByOwner) < len(owners) && i < 1024; i++ {
candidate := fmt.Sprintf("convergence-%d-%d", attempt, i)
primary := ring.GetPrimary(candidate)
if _, exists := keysByOwner[primary]; !exists {
keysByOwner[primary] = candidate
}
}
keys := make([]string, 0, len(keysByOwner))
for _, k := range keysByOwner {
keys = append(keys, k)
}
return keys
}
// checkLockMutualExclusion acquires a lock via filer0, then tries the same lock via filer1.
// Returns true if mutual exclusion holds (second attempt is denied).
func (c *distributedLockCluster) checkLockMutualExclusion(testKey string) (bool, error) {
conn0, err := grpc.NewClient(c.filerGRPCAddress(0), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return false, err
}
defer conn0.Close()
conn1, err := grpc.NewClient(c.filerGRPCAddress(1), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return false, err
}
defer conn1.Close()
client0 := filer_pb.NewSeaweedFilerClient(conn0)
client1 := filer_pb.NewSeaweedFilerClient(conn1)
// Acquire lock via filer0
ctx0, cancel0 := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel0()
resp0, err := client0.DistributedLock(ctx0, &filer_pb.LockRequest{
Name: testKey,
SecondsToLock: 5,
Owner: "convergence-filer0",
})
if err != nil || resp0.RenewToken == "" {
return false, fmt.Errorf("filer0 lock failed: err=%v resp=%v", err, resp0)
}
defer func() {
// Always release the lock we acquired
client0.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
Name: testKey,
RenewToken: resp0.RenewToken,
})
}()
// Try the same lock via filer1 - should be denied
ctx1, cancel1 := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel1()
resp1, err := client1.DistributedLock(ctx1, &filer_pb.LockRequest{
Name: testKey,
SecondsToLock: 5,
Owner: "convergence-filer1",
})
if err != nil {
return false, err
}
if resp1.RenewToken != "" {
// Both filers granted the lock - ring not converged. Release the second lock.
client1.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
Name: testKey,
RenewToken: resp1.RenewToken,
})
return false, nil
}
// Verify the denial is specifically because the lock is already held,
// not due to a transient error that might give a false positive.
if !strings.Contains(resp1.Error, "lock already owned") {
return false, nil
}
return true, nil
}
func (c *distributedLockCluster) waitForHTTP(url string, timeout time.Duration) error {
client := &net.Dialer{Timeout: time.Second}
httpClient := &httpClientWithDialer{dialer: client}
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if err := httpClient.Get(url); err == nil {
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("timed out waiting for http %s", url)
}
func (c *distributedLockCluster) waitForTCP(addr string, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
conn, err := net.DialTimeout("tcp", addr, time.Second)
if err == nil {
_ = conn.Close()
return nil
}
time.Sleep(200 * time.Millisecond)
}
return fmt.Errorf("timed out waiting for tcp %s", addr)
}
func (c *distributedLockCluster) openLog(name string) (*os.File, error) {
f, err := os.Create(filepath.Join(c.logsDir, name))
if err != nil {
return nil, err
}
c.logFiles = append(c.logFiles, f)
return f, nil
}
func (c *distributedLockCluster) tailLog(name string) string {
f, err := os.Open(filepath.Join(c.logsDir, name))
if err != nil {
return ""
}
defer f.Close()
scanner := bufio.NewScanner(f)
lines := make([]string, 0, 40)
for scanner.Scan() {
lines = append(lines, scanner.Text())
if len(lines) > 40 {
lines = lines[1:]
}
}
return strings.Join(lines, "\n")
}
func stopProcess(cmd *exec.Cmd) {
if cmd == nil || cmd.Process == nil {
return
}
_ = cmd.Process.Signal(os.Interrupt)
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()
select {
case <-done:
case <-time.After(10 * time.Second):
_ = cmd.Process.Kill()
<-done
}
}
type httpClientWithDialer struct {
dialer *net.Dialer
}
func (h *httpClientWithDialer) Get(url string) error {
client := &http.Client{
Timeout: time.Second,
Transport: &http.Transport{
DialContext: h.dialer.DialContext,
},
}
resp, err := client.Get(url)
if err != nil {
return err
}
_ = resp.Body.Close()
return nil
}