Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

4.4 KB
Raw Blame
package shell

import (
	"io"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
)

func testFixLocation(dc, rack, id string, maxVolumes int64) location {
	return location{
		dc:   dc,
		rack: rack,
		dataNode: &master_pb.DataNodeInfo{
			Id: id,
			DiskInfos: map[string]*master_pb.DiskInfo{
				"": {MaxVolumeCount: maxVolumes, FreeVolumeCount: maxVolumes},
			},
		},
	}
}

func TestReserveTargetSpreadsAcrossServers(t *testing.T) {
	src := testFixLocation("dc1", "r1", "dn0", 0)
	allLocations := []location{
		src,
		testFixLocation("dc1", "r1", "dn1", 10),
		testFixLocation("dc1", "r1", "dn2", 5),
		testFixLocation("dc1", "r1", "dn3", 3),
	}
	// keepDataNodesSorted reorders allLocations in place, so keep stable
	// per-node copies; the shared dataNode pointers carry the accounting
	byId := make(map[string]*location)
	for _, loc := range allLocations {
		loc := loc
		byId[loc.dataNode.Id] = &loc
	}
	replicas := []*VolumeReplica{
		{location: &src, info: &master_pb.VolumeInformationMessage{Id: 1}},
	}
	rp, _ := super_block.NewReplicaPlacementFromString("001")

	// with all copies in flight, consecutive reservations must not converge on
	// the emptiest server
	s := newVolumeCopyScheduler(1)
	var reserved []*location
	got := make(map[string]bool)
	for i := 0; i < 3; i++ {
		dst := s.reserveTarget(rp, replicas, allLocations, "", true)
		if dst == nil {
			t.Fatalf("reservation %d found no destination", i)
		}
		reserved = append(reserved, dst)
		got[dst.dataNode.Id] = true
	}
	for _, id := range []string{"dn1", "dn2", "dn3"} {
		if !got[id] {
			t.Errorf("expected a reservation on %s, got %v", id, got)
		}
	}

	// every eligible destination is at the copy cap: the next reservation
	// waits for a free slot instead of failing
	done := make(chan *location)
	go func() {
		done <- s.reserveTarget(rp, replicas, allLocations, "", true)
	}()
	select {
	case dst := <-done:
		t.Fatalf("reserveTarget should wait while all destinations are at the copy cap, got %s", dst.dataNode.Id)
	case <-time.After(100 * time.Millisecond):
	}
	s.releaseTarget(byId["dn1"], "", true)
	var waited *location
	select {
	case waited = <-done:
	case <-time.After(5 * time.Second):
		t.Fatal("reserveTarget did not wake up after a copy slot freed")
	}
	if waited == nil || waited.dataNode.Id != "dn1" {
		t.Fatalf("expected the freed dn1 to take the waiting copy, got %+v", waited)
	}

	// a successful copy keeps its volume slot, a failed one returns it
	s.releaseTarget(waited, "", true)
	if count := byId["dn1"].dataNode.DiskInfos[""].VolumeCount; count != 2 {
		t.Errorf("dn1 should keep 2 reserved slots, got %d", count)
	}
	for _, dst := range reserved {
		if dst.dataNode.Id != "dn1" {
			s.releaseTarget(dst, "", false)
			if count := dst.dataNode.DiskInfos[""].VolumeCount; count != 0 {
				t.Errorf("%s should have its failed reservation returned, got volume count %d", dst.dataNode.Id, count)
			}
		}
	}
	if len(s.inflight) != 0 {
		t.Errorf("all copies released, but %d still in flight", len(s.inflight))
	}
}

func TestFixUnderReplicatedVolumesInParallel(t *testing.T) {
	src := testFixLocation("dc1", "r1", "dn0", 0)
	allLocations := []location{
		src,
		testFixLocation("dc1", "r1", "dn1", 4),
		testFixLocation("dc1", "r1", "dn2", 4),
		testFixLocation("dc1", "r1", "dn3", 4),
	}
	rp, _ := super_block.NewReplicaPlacementFromString("001")

	volumeReplicas := make(map[uint32][]*VolumeReplica)
	var volumeIds []uint32
	for vid := uint32(1); vid <= 12; vid++ {
		volumeReplicas[vid] = []*VolumeReplica{
			{location: &src, info: &master_pb.VolumeInformationMessage{Id: vid, ReplicaPlacement: uint32(rp.Byte())}},
		}
		volumeIds = append(volumeIds, vid)
	}

	c := &commandVolumeFixReplication{}
	fixedVolumes, err := c.fixUnderReplicatedVolumes(nil, io.Discard, false, volumeIds, volumeReplicas, allLocations, 0, 0, 8, 1)
	if err != nil {
		t.Fatalf("fixUnderReplicatedVolumes: %v", err)
	}
	if len(fixedVolumes) != 0 {
		t.Errorf("simulation should not record fixed volumes, got %d", len(fixedVolumes))
	}

	// 12 volumes must exactly fill the 3x4 free slots without over-reserving
	// any single destination
	for _, loc := range allLocations {
		if loc.dataNode.Id == "dn0" {
			continue
		}
		diskInfo := loc.dataNode.DiskInfos[""]
		if diskInfo.VolumeCount != 4 || diskInfo.FreeVolumeCount != 0 {
			t.Errorf("%s expected exactly 4 reserved slots, got volume count %d, free %d",
				loc.dataNode.Id, diskInfo.VolumeCount, diskInfo.FreeVolumeCount)
		}
	}
}