Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

rust volume: score EC shard placement with the volume's own ratio (#11862)

Chris Lu6936bbdc3e51
diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs
index efd86474aa..4536594200 100644
--- a/seaweed-volume/src/server/grpc_server.rs
+++ b/seaweed-volume/src/server/grpc_server.rs
@@ -16,7 +16,7 @@ use crate::pb::filer_pb;
use crate::pb::master_pb;
use crate::pb::volume_server_pb;
use crate::pb::volume_server_pb::volume_server_server::VolumeServer;
-use crate::storage::erasure_coding::ec_shard::{DATA_SHARDS_COUNT, ShardId, shard_id_try_from};
+use crate::storage::erasure_coding::ec_shard::{ShardId, shard_id_try_from};
use crate::storage::erasure_coding::ecj_merge::EcjIdDecoder;
use crate::storage::needle::needle::{self, Needle};
use crate::storage::types::*;
@@ -2317,9 +2317,9 @@ impl VolumeServer for VolumeGrpcService {
// owns the .ecx on disk (volume not yet mounted — relevant
// when shards stream in mid-rebuild before any
// VolumeEcShardsMount has happened; see #9212), then any
- // HDD, then any disk. Pass the build's default data-shard
- // count for free-slot maths; the helper takes it as a
- // parameter so custom-ratio builds can swap it.
+ // HDD, then any disk. The free-slot maths uses the
+ // volume's own data-shard count from its .vif so a
+ // custom-ratio volume is not scored as 10+4.
let vid = VolumeId(info.volume_id);
let dir = if info.disk_id > 0 {
let count = store.locations.len();
@@ -2334,11 +2334,13 @@ impl VolumeServer for VolumeGrpcService {
} else {
// The mounted-volume refusal above means no disk
// holds an in-memory claim here.
+ let data_shards =
+ store.lookup_ec_data_shards(&info.collection, vid);
store
.find_ec_shard_target_location(
&info.collection,
vid,
- DATA_SHARDS_COUNT as u32,
+ data_shards,
&[],
)
.map(|i| store.locations[i].directory.clone())
@@ -3288,9 +3290,9 @@ impl VolumeServer for VolumeGrpcService {
// volume mounted, then a disk that owns the .ecx on disk (volume
// not yet mounted — relevant for ec.rebuild, where only the
// first shard carries .ecx and subsequent shards must land on
- // the same disk; see #9212), then any HDD, then any disk. Pass
- // the build's default data-shard count; the helper takes it as a
- // parameter so custom-ratio builds can swap it.
+ // the same disk; see #9212), then any HDD, then any disk. The
+ // free-slot maths uses the volume's own data-shard count from its
+ // .vif so a custom-ratio volume is not scored as 10+4.
let (dest_dir, dest_idx_dir) = {
let store = self.state.store.read().unwrap();
let count = store.locations.len();
@@ -3322,10 +3324,11 @@ impl VolumeServer for VolumeGrpcService {
req.volume_id, shard_ids, dirs
)));
}
+ let data_shards = store.lookup_ec_data_shards(&req.collection, vid);
match store.find_ec_shard_target_location(
&req.collection,
vid,
- DATA_SHARDS_COUNT as u32,
+ data_shards,
&shard_ids,
) {
Some(i) => {
diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs
index a7f0ab5b4b..014705c9f7 100644
--- a/seaweed-volume/src/storage/store.rs
+++ b/seaweed-volume/src/storage/store.rs
@@ -14,7 +14,7 @@ use crate::pb::master_pb;
use crate::storage::disk_location::DiskLocation;
use crate::storage::erasure_coding::ec_shard::{EcVolumeShard, MAX_SHARD_COUNT, ShardId};
use crate::storage::erasure_coding::ec_volume::{
- ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file,
+ ECJ_COMPACT_TMP_EXT, EcVolume, is_usable_ecx_file, read_ec_shard_config_across_dirs,
};
use crate::storage::needle::needle::{Needle, get_actual_size};
use crate::storage::needle_map::NeedleMapKind;
@@ -416,6 +416,39 @@ impl Store {
.collect()
}
+ /// Look up the per-volume EC data-shard count, preferring a mounted EC
+ /// volume's own config and then the `.vif` on whichever disk holds it.
+ /// Falls back to the default when neither exists yet (e.g. before the
+ /// first shard streams in during ec.rebuild), so
+ /// `find_ec_shard_target_location`'s free-slot maths scores custom-ratio
+ /// volumes against their real width instead of the build default.
+ pub fn lookup_ec_data_shards(&self, collection: &str, vid: VolumeId) -> u32 {
+ use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
+
+ for loc in &self.locations {
+ if let Some(ec_vol) = loc.find_ec_volume(vid) {
+ return ec_vol.data_shards;
+ }
+ }
+ let Some(first) = self.locations.first() else {
+ return DATA_SHARDS_COUNT as u32;
+ };
+ let other_dirs: Vec<String> = self.locations[1..]
+ .iter()
+ .flat_map(|loc| [loc.directory.clone(), loc.idx_directory.clone()])
+ .filter(|d| !d.is_empty())
+ .collect();
+ read_ec_shard_config_across_dirs(
+ &first.directory,
+ &first.idx_directory,
+ &other_dirs,
+ collection,
+ vid,
+ )
+ .map(|(data_shards, _, _)| data_shards)
+ .unwrap_or(DATA_SHARDS_COUNT as u32)
+ }
+
/// Create a new volume, placing it on the location with the most free space.
pub fn add_volume(
&mut self,
@@ -2980,6 +3013,35 @@ mod tests {
);
}
+ /// A volume whose `.vif` declares a custom EC ratio must feed its own
+ /// data-shard count into the free-slot maths — not the build default —
+ /// wherever the `.vif` happens to sit. With no `.vif` anywhere (a
+ /// brand-new volume mid-ec.rebuild) the answer is the default.
+ #[test]
+ fn test_lookup_ec_data_shards_reads_vif_on_any_disk() {
+ use crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT;
+
+ let (store, _tmp) = make_ec_target_test_store(3);
+ let collection = "custom-ratio";
+ let vid = VolumeId(4242);
+
+ assert_eq!(
+ store.lookup_ec_data_shards(collection, vid),
+ DATA_SHARDS_COUNT as u32,
+ "no .vif anywhere must answer the default"
+ );
+
+ // A 6+2 .vif lands on disk 2 only — the rebuild staging disk.
+ let base = volume_file_name(&store.locations[2].directory, collection, vid);
+ std::fs::write(
+ format!("{}.vif", base),
+ br#"{"ecShardConfig":{"dataShards":6,"parityShards":2}}"#,
+ )
+ .unwrap();
+
+ assert_eq!(store.lookup_ec_data_shards(collection, vid), 6);
+ }
+
/// Deleting a disk's last shard removes that disk's now-orphaned `.ecsum`;
/// a disk this delete never changed keeps its sidecar. Disk 0 models
/// ec.rebuild's index-prep state (`.ecsum` landed, no shards yet) — an