Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

20.7 KB
Raw Blame
package cluster

import (
	"context"
	"fmt"
	"net"
	"sync/atomic"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

// The gateway must resolve a lock key to the same primary the filers do,
// otherwise it dials the wrong filer and the lock still gets forwarded. Both
// sides use the same HashRing over the same server set, so for every key the
// client's hostForKey must equal the filer ring's GetPrimary.
func TestLockClientHostMatchesFilerRing(t *testing.T) {
	servers := []pb.ServerAddress{
		"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888",
	}

	filerRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
	filerRing.SetServers(servers)

	lc := NewLockClient(nil, "seed:8888")
	lc.SetRing(servers, 1)

	for _, key := range []string{
		"s3.object.write:/buckets/b/obj-0",
		"s3.object.write:/buckets/b/obj-1",
		"s3.object.write:/buckets/b/obj-2",
		"s3.object.write:/buckets/gosbench-0/w0obj-kilo-0877",
		"some/other/key",
	} {
		if got, want := lc.hostForKey(key), filerRing.GetPrimary(key); got != want {
			t.Errorf("key %q: client host %q != filer primary %q", key, got, want)
		}
	}
}

// Without a ring view, the client falls back to the seed filer (which the filer
// forwards from), preserving the pre-optimization behavior.
func TestLockClientHostFallsBackToSeed(t *testing.T) {
	lc := NewLockClient(nil, "seed:8888")
	if got := lc.hostForKey("any-key"); got != "seed:8888" {
		t.Errorf("expected seed fallback, got %q", got)
	}

	// An empty ring (no members yet) also falls back to the seed.
	lc.SetRing(nil, 1)
	if got := lc.hostForKey("any-key"); got != "seed:8888" {
		t.Errorf("expected seed fallback on empty ring, got %q", got)
	}
}

// A stale (older-version) update must not regress a newer ring view, while
// version 0 always applies as a bootstrap.
func TestLockClientSetRingVersionGuard(t *testing.T) {
	lc := NewLockClient(nil, "seed:8888")

	newer := []pb.ServerAddress{"filer-a:8888", "filer-b:8888"}
	lc.SetRing(newer, 10)
	primaryAt10 := lc.hostForKey("k")

	// Older version is ignored.
	lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 5)
	if got := lc.hostForKey("k"); got != primaryAt10 {
		t.Errorf("stale update applied: host changed to %q", got)
	}

	// version 0 is always accepted.
	lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 0)
	if got := lc.hostForKey("k"); got != "filer-z:8888" {
		t.Errorf("bootstrap update not applied, got %q", got)
	}
}

// PrimaryForKey returns "" before any ring is received (so a route-by-key
// caller falls back to the distributed lock) and the ring owner afterwards,
// unlike hostForKey which falls back to the seed.
func TestLockClientPrimaryForKey(t *testing.T) {
	lc := NewLockClient(nil, "seed:8888")
	if got := lc.PrimaryForKey("k"); got != "" {
		t.Errorf("expected empty before ring, got %q", got)
	}

	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888"}, 1)
	got := lc.PrimaryForKey("k")
	if got == "" {
		t.Fatal("expected an owner after ring set")
	}
	if got != lc.hostForKey("k") {
		t.Errorf("PrimaryForKey %q disagrees with hostForKey %q", got, lc.hostForKey("k"))
	}
}

// A moved key reports its previous owner within the cooling-off window; an unmoved
// key reports none.
func TestLockClientPriorOwnerForKey(t *testing.T) {
	lc := NewLockClient(nil, "seed:8888")

	setA := []pb.ServerAddress{"filer-a:8888", "filer-b:8888", "filer-c:8888"}
	lc.SetRing(setA, 1)
	// One ring: nothing to fall back to.
	if got := lc.PriorOwnerForKey("any"); got != "" {
		t.Fatalf("single ring should have no prior owner, got %q", got)
	}

	priorRing := lock_manager.NewHashRing(lock_manager.DefaultVnodeCount)
	priorRing.SetServers(setA)

	// Add a server so some keys' ownership moves.
	setB := []pb.ServerAddress{"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888"}
	lc.SetRing(setB, 2)

	var moved, stable string
	for i := 0; i < 2000 && (moved == "" || stable == ""); i++ {
		key := fmt.Sprintf("key-%d", i)
		if lc.PrimaryForKey(key) != priorRing.GetPrimary(key) {
			if moved == "" {
				moved = key
			}
		} else if stable == "" {
			stable = key
		}
	}
	if moved == "" || stable == "" {
		t.Skip("could not find both a moved and a stable key")
	}

	if got, want := lc.PriorOwnerForKey(moved), priorRing.GetPrimary(moved); got != want {
		t.Fatalf("PriorOwnerForKey(moved)=%q, want %q", got, want)
	}
	if got := lc.PriorOwnerForKey(stable); got != "" {
		t.Fatalf("unmoved key should have no prior owner, got %q", got)
	}
}

// The prior owner is only offered within the cooling-off window.
func TestLockClientPriorOwnerForKeyExpires(t *testing.T) {
	lc := NewLockClient(nil, "seed:8888")
	lc.priorWindow = 20 * time.Millisecond

	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888", "filer-c:8888"}, 1)
	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888", "filer-c:8888", "filer-d:8888"}, 2)

	time.Sleep(40 * time.Millisecond)
	for i := 0; i < 2000; i++ {
		if got := lc.PriorOwnerForKey(fmt.Sprintf("key-%d", i)); got != "" {
			t.Fatalf("prior owner should expire after the cooling window, got %q", got)
		}
	}
}

// A master change clears the version gate so the new leader's (lower-versioned)
// snapshot applies — versions are only comparable within one master's stream.
func TestLockClientResetRing(t *testing.T) {
	lc := NewLockClient(nil, "seed:8888")

	lc.SetRing([]pb.ServerAddress{"filer-a:8888", "filer-b:8888"}, 100)
	lc.ResetRing()

	// The last ring keeps routing during the gap; only version acceptance
	// is reset so the new leader's (lower-versioned) snapshot applies.
	if got := lc.hostForKey("k"); got == "seed:8888" {
		t.Fatal("expected the previous ring to keep routing after reset")
	}
	lc.SetRing([]pb.ServerAddress{"filer-z:8888"}, 50)
	if got := lc.hostForKey("k"); got != "filer-z:8888" {
		t.Fatalf("lower version from new master not applied, got %q", got)
	}
}

type noLockServerFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
}

func (s *noLockServerFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
	return &filer_pb.LockResponse{Error: lock_manager.NoLockServerError.Error()}, nil
}

// When every filer reports an empty lock ring, lock acquisition must fail
// after a bounded period instead of hanging the write forever.
func TestNewShortLivedLockFailsFastOnNoLockServer(t *testing.T) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(grpcServer, &noLockServerFiler{})
	go grpcServer.Serve(listener)
	defer grpcServer.Stop()

	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
	host, port, err := net.SplitHostPort(listener.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	// "host:httpPort.grpcPort" dials the fake filer's port directly.
	lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
	lc.noLockServerRetryPeriod = 200 * time.Millisecond

	start := time.Now()
	lock := lc.NewShortLivedLock("test-key", "test-owner")
	elapsed := time.Since(start)

	if lock != nil {
		t.Fatal("expected nil lock when no lock server exists")
	}
	if elapsed > 10*time.Second {
		t.Fatalf("lock acquisition took %v, expected fail-fast", elapsed)
	}
}

type contendedLockFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
}

func (s *contendedLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
	return &filer_pb.LockResponse{Error: "lock already owned by someone"}, nil
}

func newContendedLockClient(t *testing.T) *LockClient {
	t.Helper()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(grpcServer, &contendedLockFiler{})
	go grpcServer.Serve(listener)
	t.Cleanup(grpcServer.Stop)

	host, port, err := net.SplitHostPort(listener.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	return NewLockClient(grpc.WithTransportCredentials(insecure.NewCredentials()), pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
}

// A lock held by another owner is ordinary contention: acquisition waits it
// out rather than failing on the unavailability bound.
func TestNewShortLivedLockWaitsOutContention(t *testing.T) {
	lc := newContendedLockClient(t)
	lc.noLockServerRetryPeriod = 200 * time.Millisecond

	done := make(chan *LiveLock, 1)
	go func() {
		done <- lc.NewShortLivedLock("test-key", "test-owner")
	}()
	select {
	case <-done:
		t.Fatal("ordinary lock contention must not hit the unavailability bound")
	case <-time.After(3 * lc.noLockServerRetryPeriod):
	}
}

func TestNewBlockingLongLivedLockWithinGivesUpOnContention(t *testing.T) {
	lc := newContendedLockClient(t)

	maxWait := 1500 * time.Millisecond
	start := time.Now()
	lock, err := lc.NewBlockingLongLivedLockWithin("test-key", "test-owner", 15*time.Second, maxWait)
	elapsed := time.Since(start)

	if lock != nil || err == nil {
		t.Fatalf("got lock %v err %v, want no lock and the contention error", lock, err)
	}
	if elapsed < maxWait || elapsed > maxWait+5*time.Second {
		t.Fatalf("gave up after %v, want just past %v", elapsed, maxWait)
	}
}

// A ring member that refuses connections is unavailability, not contention:
// acquisition fails on the same bound as "no lock server found".
func TestNewShortLivedLockFailsFastOnUnreachableFiler(t *testing.T) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	addr := listener.Addr().String()
	listener.Close()

	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
	host, port, err := net.SplitHostPort(addr)
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	lc := NewLockClient(dialOption, pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
	lc.noLockServerRetryPeriod = 200 * time.Millisecond

	start := time.Now()
	lock := lc.NewShortLivedLock("test-key", "test-owner")
	elapsed := time.Since(start)

	if lock != nil {
		t.Fatal("expected nil lock when the ring member is unreachable")
	}
	if elapsed > 10*time.Second {
		t.Fatalf("lock acquisition took %v, expected fail-fast", elapsed)
	}
}

type hangingLockFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
}

func (s *hangingLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
	<-ctx.Done()
	return nil, ctx.Err()
}

type grantingLockFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
}

func (s *grantingLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
	return &filer_pb.LockResponse{RenewToken: "tok"}, nil
}

// A ring member that accepts connections but never answers must not swallow
// the retry window: when its budget is spent the seed route still gets an
// attempt before acquisition gives up.
func TestNewShortLivedLockFallsBackToSeedOnHangingPrimary(t *testing.T) {
	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())

	hangListener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	hangServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(hangServer, &hangingLockFiler{})
	go hangServer.Serve(hangListener)
	defer hangServer.Stop()

	seedListener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	seedServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(seedServer, &grantingLockFiler{})
	go seedServer.Serve(seedListener)
	defer seedServer.Stop()

	hangHost, hangPort, err := net.SplitHostPort(hangListener.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	seedHost, seedPort, err := net.SplitHostPort(seedListener.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	// ServerAddress.Equals compares the http port, so the two fake filers
	// need distinct ones even though only the grpc port is dialed.
	seedAddr := pb.ServerAddress(fmt.Sprintf("%s:1.%s", seedHost, seedPort))
	primaryAddr := pb.ServerAddress(fmt.Sprintf("%s:2.%s", hangHost, hangPort))

	lc := NewLockClient(dialOption, seedAddr)
	lc.lockRpcTimeout = 50 * time.Millisecond
	lc.noLockServerRetryPeriod = 200 * time.Millisecond
	lc.SetRing([]pb.ServerAddress{primaryAddr}, 1)

	start := time.Now()
	lock := lc.NewShortLivedLock("test-key", "test-owner")
	elapsed := time.Since(start)

	if lock == nil {
		t.Fatal("expected the seed fallback to acquire the lock")
	}
	if !lock.hostFiler.Equals(seedAddr) {
		t.Fatalf("expected acquisition through the seed %s, got %s", seedAddr, lock.hostFiler)
	}
	if elapsed > 5*time.Second {
		t.Fatalf("lock acquisition took %v, expected it bounded by the per-route window", elapsed)
	}
}

type redirectingLockFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
	movedTo string
}

func (s *redirectingLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
	return &filer_pb.LockResponse{Error: "forward failed", LockHostMovedTo: s.movedTo}, nil
}

// The seed can redirect back to a dead primary via LockHostMovedTo. The seed
// fallback gets one fresh window per acquisition — letting every return to
// the seed refill it would make the primary->seed->primary cycle unbounded.
func TestNewShortLivedLockBoundsSeedRedirectCycle(t *testing.T) {
	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())

	hangListener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	hangServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(hangServer, &hangingLockFiler{})
	go hangServer.Serve(hangListener)
	defer hangServer.Stop()

	hangHost, hangPort, err := net.SplitHostPort(hangListener.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	primaryAddr := pb.ServerAddress(fmt.Sprintf("%s:2.%s", hangHost, hangPort))

	seedListener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	seedServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(seedServer, &redirectingLockFiler{movedTo: string(primaryAddr)})
	go seedServer.Serve(seedListener)
	defer seedServer.Stop()

	seedHost, seedPort, err := net.SplitHostPort(seedListener.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	seedAddr := pb.ServerAddress(fmt.Sprintf("%s:1.%s", seedHost, seedPort))

	lc := NewLockClient(dialOption, seedAddr)
	lc.lockRpcTimeout = 50 * time.Millisecond
	lc.noLockServerRetryPeriod = 200 * time.Millisecond

	start := time.Now()
	lock := lc.NewShortLivedLock("test-key", "test-owner")
	elapsed := time.Since(start)

	if lock != nil {
		t.Fatal("expected nil lock when the seed keeps redirecting to a dead primary")
	}
	if elapsed > 5*time.Second {
		t.Fatalf("lock acquisition took %v, expected a bounded redirect cycle", elapsed)
	}
}

type cleanRedirectLockFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
	movedTo string
}

func (s *cleanRedirectLockFiler) DistributedLock(ctx context.Context, req *filer_pb.LockRequest) (*filer_pb.LockResponse, error) {
	return &filer_pb.LockResponse{LockHostMovedTo: s.movedTo}, nil
}

// A second hop whose ring disagrees answers with a bare LockHostMovedTo and
// no error, so the first hop relays a clean redirect. Two filers pointing at
// each other then bounce the client forever if every redirect refills the
// window — tokenless redirects must count against it like failures do.
func TestNewShortLivedLockBoundsCleanRedirectCycle(t *testing.T) {
	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())

	listenerA, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	listenerB, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	hostA, portA, err := net.SplitHostPort(listenerA.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	hostB, portB, err := net.SplitHostPort(listenerB.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	addrA := pb.ServerAddress(fmt.Sprintf("%s:1.%s", hostA, portA))
	addrB := pb.ServerAddress(fmt.Sprintf("%s:2.%s", hostB, portB))

	serverA := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(serverA, &cleanRedirectLockFiler{movedTo: string(addrB)})
	go serverA.Serve(listenerA)
	defer serverA.Stop()
	serverB := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(serverB, &cleanRedirectLockFiler{movedTo: string(addrA)})
	go serverB.Serve(listenerB)
	defer serverB.Stop()

	lc := NewLockClient(dialOption, addrA)
	lc.lockRpcTimeout = 50 * time.Millisecond
	lc.noLockServerRetryPeriod = 200 * time.Millisecond

	done := make(chan *LiveLock, 1)
	go func() {
		done <- lc.NewShortLivedLock("test-key", "test-owner")
	}()
	select {
	case lock := <-done:
		if lock != nil {
			t.Fatal("expected nil lock when the ring keeps redirecting")
		}
	case <-time.After(5 * time.Second):
		t.Fatal("clean redirect cycle was not bounded by the retry window")
	}
}

// A failed attempt keeps the token it sent: the request may have renewed the
// lease server-side before its response was lost, and a retry with the last
// token can still succeed. Only a definitive rejection may clear it.
func TestDoLockKeepsRenewTokenOnError(t *testing.T) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	addr := listener.Addr().String()
	listener.Close()

	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
	host, port, err := net.SplitHostPort(addr)
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	filerAddr := pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port))
	lc := NewLockClient(dialOption, filerAddr)
	lock := &LiveLock{
		key:            "test-key",
		renewToken:     "tok",
		hostFiler:      filerAddr,
		grpcDialOption: dialOption,
		lc:             lc,
	}
	if _, _, err := lock.doLock(time.Second, 50*time.Millisecond); err == nil {
		t.Fatal("expected the attempt to fail against a dead member")
	}
	if lock.renewToken != "tok" {
		t.Errorf("renewToken = %q, want it kept for the next attempt", lock.renewToken)
	}
}

// A definitive rejection answers with an empty token, which clears the stale
// one — only transport failures keep it.
func TestDoLockClearsRenewTokenOnRejection(t *testing.T) {
	lc := newContendedLockClient(t)
	lock := &LiveLock{
		key:            "test-key",
		renewToken:     "tok",
		hostFiler:      lc.seedFiler,
		grpcDialOption: lc.grpcDialOption,
		lc:             lc,
	}
	errorMessage, _, err := lock.doLock(time.Second, time.Second)
	if err != nil || errorMessage == "" {
		t.Fatalf("expected a contended attempt to report an error message, got errorMessage=%q err=%v", errorMessage, err)
	}
	if lock.renewToken != "" {
		t.Errorf("renewToken = %q, want it cleared by the rejection", lock.renewToken)
	}
}

// A bare LockHostMovedTo redirect is not a verdict on the token: the
// request may have committed a renewal before the ring moved, so the held
// token survives to renew against the new host.
func TestDoLockKeepsRenewTokenOnCleanRedirect(t *testing.T) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	grpcServer := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(grpcServer, &cleanRedirectLockFiler{movedTo: "127.0.0.1:1.1"})
	go grpcServer.Serve(listener)
	defer grpcServer.Stop()

	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
	host, port, err := net.SplitHostPort(listener.Addr().String())
	if err != nil {
		t.Fatalf("split host port: %v", err)
	}
	filerAddr := pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port))
	lc := NewLockClient(dialOption, filerAddr)
	lock := &LiveLock{
		key:            "test-key",
		renewToken:     "tok",
		hostFiler:      filerAddr,
		grpcDialOption: dialOption,
		lc:             lc,
	}
	errorMessage, _, err := lock.doLock(time.Second, time.Second)
	if err != nil || errorMessage != "" {
		t.Fatalf("expected a clean redirect, got errorMessage=%q err=%v", errorMessage, err)
	}
	if lock.renewToken != "tok" {
		t.Errorf("renewToken = %q, want it kept across the redirect", lock.renewToken)
	}
}

// LiveLock.generation is a fencing token written and read with 64-bit atomic
// operations. On 32-bit platforms (GOARCH=386 and GOARCH=arm) a 64-bit atomic
// op requires an 8-byte-aligned address, which Go only guarantees for the
// first field of an allocated struct. generation must therefore stay first:
// a misaligned field makes StoreInt64 panic with "unaligned 64-bit atomic
// operation" the moment this test runs on a 32-bit target.
func TestLiveLockGenerationIsAligned(t *testing.T) {
	var lock LiveLock
	atomic.StoreInt64(&lock.generation, 42)
	if got := lock.Generation(); got != 42 {
		t.Fatalf("generation = %d, want 42", got)
	}
}