Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

7.7 KB
Raw Blame
package topology

import (
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
)

func (dn *DataNode) GetEcShards() (ret []*erasure_coding.EcVolumeInfo) {
	dn.RLock()
	for _, c := range dn.children {
		disk := c.(*Disk)
		ret = append(ret, disk.GetEcShards()...)
	}
	dn.RUnlock()
	return ret
}

// ecShardKey identifies a per-physical-disk EC shard entry on a DataNode.
// A single volume's EC shards can live on multiple physical disks of one
// DataNode (e.g. a 10+4 volume spread across 4 mount points), so entries
// must be tracked per (volume, physical disk) rather than per volume alone.
// See issue #9212.
type ecShardKey struct {
	vid    needle.VolumeId
	diskId types.DiskId
}

func (dn *DataNode) UpdateEcShards(actualShards []*erasure_coding.EcVolumeInfo) (newShards, deletedShards []*erasure_coding.EcVolumeInfo) {
	// Aggregate incoming messages by (volume, physical disk). Duplicates for
	// the same (vid, diskId) are merged; entries for the same vid on
	// different disks stay separate so per-physical-disk attribution is
	// preserved all the way to DiskInfo.EcShardInfos — which the admin
	// shell's ec.balance / ec.rebuild planners read via eci.DiskId.
	actualByKey := make(map[ecShardKey]*erasure_coding.EcVolumeInfo, len(actualShards))
	for _, ecShards := range actualShards {
		key := ecShardKey{vid: ecShards.VolumeId, diskId: types.DiskId(ecShards.DiskId)}
		if existing, ok := actualByKey[key]; ok {
			existing.ShardsInfo.Add(ecShards.ShardsInfo)
			continue
		}
		// Clone so subsequent merges for the same key don't mutate the
		// caller's EcVolumeInfo, and so the diff below is stable.
		clone := *ecShards
		clone.ShardsInfo = ecShards.ShardsInfo.Copy()
		actualByKey[key] = &clone
	}

	// Hold dn.Lock for the full read-diff-write cycle so concurrent heartbeats,
	// getOrCreateDisk calls, and UpAdjustDiskUsageDelta updates on this data
	// node are serialized with us — matching UpdateVolumes' locking model.
	// Internal helpers below assume the caller holds dn.Lock.
	dn.Lock()
	defer dn.Unlock()

	existingEcShards := dn.getEcShardsLocked()

	// find out the newShards and deletedShards
	for _, ecShards := range existingEcShards {

		var newShardCount, deletedShardCount int
		disk := dn.getOrCreateDisk(ecShards.DiskType)

		key := ecShardKey{vid: ecShards.VolumeId, diskId: types.DiskId(ecShards.DiskId)}
		if actualEcShards, ok := actualByKey[key]; !ok {
			// dn registered ec shards not found in the new set of ec shards
			deletedShards = append(deletedShards, ecShards)
			deletedShardCount += ecShards.ShardsInfo.Count()
		} else {
			// found, but maybe the actual shard could be missing
			a := actualEcShards.Minus(ecShards)
			if a.ShardsInfo.Count() > 0 {
				newShards = append(newShards, a)
				newShardCount += a.ShardsInfo.Count()
			}
			d := ecShards.Minus(actualEcShards)
			if d.ShardsInfo.Count() > 0 {
				deletedShards = append(deletedShards, d)
				deletedShardCount += d.ShardsInfo.Count()
			}
			// Counts move without any shard changing hands, and the
			// rewrite below only runs on a membership change, so publish
			// the new counts on their own.
			if ecCountsChanged(ecShards, actualEcShards) {
				disk.refreshEcShardCounts(key.vid, key.diskId, actualEcShards.FileCount, actualEcShards.DeleteCount)
			}
		}

		if (newShardCount - deletedShardCount) != 0 {
			disk.UpAdjustDiskUsageDelta(types.ToDiskType(ecShards.DiskType), &DiskUsageCounts{
				ecShardCount: int64(newShardCount - deletedShardCount),
			})
		}

	}

	existingKeys := make(map[ecShardKey]struct{}, len(existingEcShards))
	for _, ev := range existingEcShards {
		existingKeys[ecShardKey{vid: ev.VolumeId, diskId: types.DiskId(ev.DiskId)}] = struct{}{}
	}
	for key, ecShards := range actualByKey {
		if _, found := existingKeys[key]; found {
			continue
		}

		newShards = append(newShards, ecShards)

		disk := dn.getOrCreateDisk(ecShards.DiskType)
		disk.UpAdjustDiskUsageDelta(types.ToDiskType(ecShards.DiskType), &DiskUsageCounts{
			ecShardCount: int64(ecShards.ShardsInfo.Count()),
		})
	}

	if len(newShards) > 0 || len(deletedShards) > 0 {
		// if changed, set to the new ec shard map
		dn.doUpdateEcShardsLocked(actualByKey)
	}

	return
}

// ecCountsChanged reports whether a heartbeat moves the stored FileCount or
// DeleteCount. A message reporting 0/0 did not count (older volume servers omit
// the fields), so it keeps the stored values and carries them onto the incoming
// record in case the map is rewritten.
func ecCountsChanged(stored, actual *erasure_coding.EcVolumeInfo) bool {
	if actual.FileCount == 0 && actual.DeleteCount == 0 {
		actual.FileCount, actual.DeleteCount = stored.FileCount, stored.DeleteCount
		return false
	}
	return actual.FileCount != stored.FileCount || actual.DeleteCount != stored.DeleteCount
}

// getEcShardsLocked returns the flat list of per-physical-disk EC shard
// entries across all children. Caller MUST hold dn.Lock (or RLock); each
// disk's ecShardsLock is taken internally via Disk.GetEcShards.
func (dn *DataNode) getEcShardsLocked() (ret []*erasure_coding.EcVolumeInfo) {
	for _, c := range dn.children {
		disk := c.(*Disk)
		ret = append(ret, disk.GetEcShards()...)
	}
	return ret
}

func (dn *DataNode) HasEcShards(volumeId needle.VolumeId) (found bool) {
	dn.RLock()
	defer dn.RUnlock()
	for _, c := range dn.children {
		disk := c.(*Disk)
		disk.ecShardsLock.RLock()
		byDisk, ok := disk.ecShards[volumeId]
		has := ok && len(byDisk) > 0
		disk.ecShardsLock.RUnlock()
		if has {
			return true
		}
	}
	return
}

// HasEcShard reports whether any disk on this node still holds the shard.
// Registration lookups key on the node alone, so a shard moving between its
// disks must not drop the node from the lookup while a copy remains.
func (dn *DataNode) HasEcShard(volumeId needle.VolumeId, shardId erasure_coding.ShardId) (found bool) {
	dn.RLock()
	defer dn.RUnlock()
	for _, c := range dn.children {
		disk := c.(*Disk)
		disk.ecShardsLock.RLock()
		for _, ev := range disk.ecShards[volumeId] {
			if ev.ShardsInfo.Has(shardId) {
				found = true
				break
			}
		}
		disk.ecShardsLock.RUnlock()
		if found {
			return true
		}
	}
	return
}

// doUpdateEcShardsLocked rewrites disk.ecShards from actualByKey. Caller
// MUST hold dn.Lock; each disk's ecShardsLock is taken internally around
// the read-modify-write of its ecShards map.
func (dn *DataNode) doUpdateEcShardsLocked(actualByKey map[ecShardKey]*erasure_coding.EcVolumeInfo) {
	for _, c := range dn.children {
		disk := c.(*Disk)
		disk.ecShardsLock.Lock()
		disk.ecShards = make(map[needle.VolumeId]map[types.DiskId]*erasure_coding.EcVolumeInfo)
		disk.ecShardsLock.Unlock()
	}
	for _, shard := range actualByKey {
		disk := dn.getOrCreateDisk(shard.DiskType)
		disk.ecShardsLock.Lock()
		byDisk, ok := disk.ecShards[shard.VolumeId]
		if !ok {
			byDisk = make(map[types.DiskId]*erasure_coding.EcVolumeInfo, 1)
			disk.ecShards[shard.VolumeId] = byDisk
		}
		byDisk[types.DiskId(shard.DiskId)] = shard
		disk.ecShardsLock.Unlock()
	}
}

func (dn *DataNode) DeltaUpdateEcShards(newShards, deletedShards []*erasure_coding.EcVolumeInfo) {
	for _, newShard := range newShards {
		dn.AddOrUpdateEcShard(newShard)
	}

	for _, deletedShard := range deletedShards {
		dn.DeleteEcShard(deletedShard)
	}

}

func (dn *DataNode) AddOrUpdateEcShard(s *erasure_coding.EcVolumeInfo) {
	disk := dn.getOrCreateDiskWithLock(s.DiskType)
	disk.AddOrUpdateEcShard(s)
}

func (dn *DataNode) DeleteEcShard(s *erasure_coding.EcVolumeInfo) {
	disk := dn.getOrCreateDiskWithLock(s.DiskType)
	disk.DeleteEcShard(s)
}

func (dn *DataNode) HasVolumesById(volumeId needle.VolumeId) (hasVolumeId bool) {

	dn.RLock()
	defer dn.RUnlock()
	for _, c := range dn.children {
		disk := c.(*Disk)
		if disk.HasVolumesById(volumeId) {
			return true
		}
	}
	return false

}