Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

ec: uniform shard block layout (#10932)

Chris Lu88c873ecd4fb
diff --git a/seaweed-volume/proto/volume_server.proto b/seaweed-volume/proto/volume_server.proto
index 6b64ef5296..961a800fe7 100644
--- a/seaweed-volume/proto/volume_server.proto
+++ b/seaweed-volume/proto/volume_server.proto
@@ -539,6 +539,7 @@ message VolumeEcShardsInfoResponse {
uint64 volume_size = 2;
uint64 file_count = 3;
uint64 file_deleted_count = 4;
+ EcShardConfig ec_shard_config = 10; // the layout this holder serves reads through; a binary predating a field reports it as unset
}
message EcShardInfo {
@@ -612,6 +613,7 @@ message EcShardConfig {
uint32 data_shards = 1; // Number of data shards (e.g., 10)
uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
int64 encode_ts_ns = 3; // encode time (unix nanos); a read served from a shard of a different encode run is rejected
+ int64 block_size = 4; // uniform block layout: each shard is a single contiguous block of this many bytes; 0 = legacy 1GiB/1MiB two-tier layout
}
// EcBitrotProtection is the entire content of a bitrot checksum sidecar
// (<base>.ecsum for the legacy generation, <base>.ecsum.v<N> for vacuum
diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs
index e366413d58..dc956c4148 100644
--- a/seaweed-volume/src/server/grpc_server.rs
+++ b/seaweed-volume/src/server/grpc_server.rs
@@ -92,6 +92,57 @@ pub fn load_state_file(
volume_server_pb::VolumeServerState::decode(data.as_slice()).ok()
}
+/// One disk location's stake in an EC volume, as seen by the rebuild handler.
+struct LocInfo {
+ dir: String,
+ idx_dir: String,
+ shard_count: usize,
+ has_ecx: bool,
+}
+
+/// Picks the location a rebuild should write into — the one holding an `.ecx`
+/// and the most shards — and returns every other directory it may have to read
+/// from. Shards are only half of what the rebuild needs: a split
+/// `-dir`/`-dir.idx` layout keeps `.ecx`/`.ecj`/`.vif` with the INDEX, and on a
+/// multi-disk server the chosen disk may hold nothing but shards while this
+/// volume's `.vif` or generation-0 `.ecsum` sits on a sibling. Miss those and
+/// the layout resolution falls back to 10+4 with the legacy striping and
+/// reconstructs through the wrong matrix, so both directories of every other
+/// location are listed. The rebuild's own two are passed separately by the
+/// caller and dropped here, along with empties and duplicates.
+///
+/// Returns `None` when no location holds an `.ecx`, i.e. there is nothing to
+/// rebuild from.
+fn select_rebuild_location(loc_infos: &[LocInfo]) -> Option<(usize, Vec<String>)> {
+ let mut rebuild_loc_idx: Option<usize> = None;
+ let mut other_dirs: Vec<String> = Vec::new();
+
+ for (i, info) in loc_infos.iter().enumerate() {
+ let better = info.has_ecx
+ && rebuild_loc_idx
+ .is_none_or(|prev| info.shard_count > loc_infos[prev].shard_count);
+ if better {
+ if let Some(prev) = rebuild_loc_idx {
+ other_dirs.push(loc_infos[prev].dir.clone());
+ other_dirs.push(loc_infos[prev].idx_dir.clone());
+ }
+ rebuild_loc_idx = Some(i);
+ } else {
+ other_dirs.push(info.dir.clone());
+ other_dirs.push(info.idx_dir.clone());
+ }
+ }
+
+ let rebuild_loc_idx = rebuild_loc_idx?;
+ let rebuild_dir = &loc_infos[rebuild_loc_idx].dir;
+ let rebuild_idx_dir = &loc_infos[rebuild_loc_idx].idx_dir;
+ other_dirs.retain(|d| !d.is_empty() && d != rebuild_dir && d != rebuild_idx_dir);
+ other_dirs.sort();
+ other_dirs.dedup();
+
+ Some((rebuild_loc_idx, other_dirs))
+}
+
struct WriteThrottler {
bytes_per_second: i64,
last_size_counter: i64,
@@ -2383,13 +2434,21 @@ impl VolumeServer for VolumeGrpcService {
)
};
- // Check existing .vif for EC shard config (matching Go's MaybeLoadVolumeInfo)
- let (data_shards, parity_shards) =
+ // Check existing .vif for EC shard config (matching Go's MaybeLoadVolumeInfo).
+ // The block size is recomputed by the encode for the current .dat, so
+ // only the ratio is carried over from a prior config.
+ let (data_shards, parity_shards, _) =
crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
&dir, &idx_dir, collection, vid,
- );
+ )
+ .map_err(|e| {
+ tonic::Status::internal(format!(
+ "read ec shard config for volume {}: {}",
+ vid.0, e
+ ))
+ })?;
- if let Err(e) = crate::storage::erasure_coding::ec_encoder::write_ec_files(
+ let block_size = match crate::storage::erasure_coding::ec_encoder::write_ec_files(
&dir,
&idx_dir,
collection,
@@ -2397,16 +2456,19 @@ impl VolumeServer for VolumeGrpcService {
data_shards as usize,
parity_shards as usize,
) {
- // Cleanup partially-created .ecNN and .ecx files on failure (matching Go defer)
- let base = crate::storage::volume::volume_file_name(&dir, collection, vid);
- let total_shards = data_shards + parity_shards;
- for i in 0..total_shards {
- let shard_path = format!("{}.ec{:02}", base, i);
- let _ = std::fs::remove_file(&shard_path);
+ Ok(block_size) => block_size,
+ Err(e) => {
+ // Cleanup partially-created .ecNN and .ecx files on failure (matching Go defer)
+ let base = crate::storage::volume::volume_file_name(&dir, collection, vid);
+ let total_shards = data_shards + parity_shards;
+ for i in 0..total_shards {
+ let shard_path = format!("{}.ec{:02}", base, i);
+ let _ = std::fs::remove_file(&shard_path);
+ }
+ let _ = std::fs::remove_file(format!("{}.ecx", base));
+ return Err(Status::internal(e.to_string()));
}
- let _ = std::fs::remove_file(format!("{}.ecx", base));
- return Err(Status::internal(e.to_string()));
- }
+ };
// Write .vif file with EC shard metadata
{
@@ -2425,6 +2487,7 @@ impl VolumeServer for VolumeGrpcService {
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos() as i64,
+ block_size,
}),
..Default::default()
};
@@ -2458,13 +2521,6 @@ impl VolumeServer for VolumeGrpcService {
format!("{}_{}", collection, vid.0)
};
- struct LocInfo {
- dir: String,
- idx_dir: String,
- shard_count: usize,
- has_ecx: bool,
- }
-
let store = self.state.store.read().unwrap();
let mut loc_infos: Vec<LocInfo> = Vec::new();
@@ -2512,26 +2568,8 @@ impl VolumeServer for VolumeGrpcService {
));
}
- // Pick rebuild location: has .ecx and most shards
- let mut rebuild_loc_idx: Option<usize> = None;
- let mut other_dirs: Vec<String> = Vec::new();
-
- for (i, info) in loc_infos.iter().enumerate() {
- if info.has_ecx
- && (rebuild_loc_idx.is_none()
- || info.shard_count > loc_infos[rebuild_loc_idx.unwrap()].shard_count)
- {
- if let Some(prev) = rebuild_loc_idx {
- other_dirs.push(loc_infos[prev].dir.clone());
- }
- rebuild_loc_idx = Some(i);
- } else {
- other_dirs.push(info.dir.clone());
- }
- }
-
- let rebuild_loc_idx = match rebuild_loc_idx {
- Some(i) => i,
+ let (rebuild_loc_idx, other_dirs) = match select_rebuild_location(&loc_infos) {
+ Some(picked) => picked,
None => {
return Ok(Response::new(
volume_server_pb::VolumeEcShardsRebuildResponse {
@@ -2545,13 +2583,37 @@ impl VolumeServer for VolumeGrpcService {
let rebuild_idx_dir = loc_infos[rebuild_loc_idx].idx_dir.clone();
// Determine data/parity shard config from rebuild dir
- let (data_shards, parity_shards) =
- crate::storage::erasure_coding::ec_volume::read_ec_shard_config(
+ // The encode-time .dat size resolves the row count the ecx rebuild
+ // de-stripes with; 0 leaves it to infer from the padded shard extent.
+ // Both lookups search the sibling disks too: the rebuild writes into one
+ // location, but a multi-disk server may keep this volume's .vif or its
+ // generation-0 .ecsum on another, and defaulting to 10+4 with the
+ // legacy layout would reconstruct through the wrong matrix.
+ let dat_file_size = crate::storage::erasure_coding::ec_volume::load_vif_info_across_dirs(
+ &rebuild_dir,
+ &rebuild_idx_dir,
+ &other_dirs,
+ collection,
+ vid,
+ )
+ .ok()
+ .flatten()
+ .map(|(v, _)| v.dat_file_size)
+ .unwrap_or(0);
+ let (data_shards, parity_shards, block_size) =
+ crate::storage::erasure_coding::ec_volume::read_ec_shard_config_across_dirs(
&rebuild_dir,
&rebuild_idx_dir,
+ &other_dirs,
collection,
vid,
- );
+ )
+ .map_err(|e| {
+ tonic::Status::internal(format!(
+ "read ec shard config for volume {}: {}",
+ vid.0, e
+ ))
+ })?;
let total_shards = data_shards + parity_shards;
// Check which shards are missing (check rebuild dir and all other dirs)
@@ -2590,8 +2652,17 @@ impl VolumeServer for VolumeGrpcService {
// Rebuild missing shards, searching all locations for input shards.
// Pass other_dirs so shards on sibling disks are found even when the
- // primary rebuild dir doesn't hold them.
- let other_dir_refs: Vec<&str> = other_dirs.iter().map(|s| s.as_str()).collect();
+ // primary rebuild dir doesn't hold them. This one takes a single flat
+ // list — the shape Go's RebuildEcFiles uses — so unlike the resolvers
+ // above it cannot be handed the rebuild's own index directory
+ // separately, and a split -dir/-dir.idx location keeps its .ecx and
+ // .vif there. Go's additionalDirs carries that directory for the same
+ // reason.
+ let mut rebuild_search_dirs: Vec<String> = other_dirs.clone();
+ if !rebuild_idx_dir.is_empty() && rebuild_idx_dir != rebuild_dir {
+ rebuild_search_dirs.push(rebuild_idx_dir.clone());
+ }
+ let other_dir_refs: Vec<&str> = rebuild_search_dirs.iter().map(|s| s.as_str()).collect();
crate::storage::erasure_coding::ec_encoder::rebuild_ec_files(
&rebuild_dir,
collection,
@@ -2629,6 +2700,8 @@ impl VolumeServer for VolumeGrpcService {
collection,
vid,
data_shards as usize,
+ block_size,
+ dat_file_size,
&ecx_dir_refs,
)
.map_err(|e| Status::internal(format!("RebuildEcxFile: {}", e)))?;
@@ -3015,6 +3088,27 @@ impl VolumeServer for VolumeGrpcService {
Status::internal(format!("mount {}.{}: {}", req.volume_id, shard_id, e))
})?;
}
+ // A delivery can bring the checksum manifest alongside the shards, but
+ // the receive path only writes the file. When this server already had
+ // the volume mounted, the EcVolume in memory keeps whatever protection
+ // state it resolved at mount — off, for a volume whose sidecar arrives
+ // now — until a remount. Re-resolve it here, where the shards it
+ // describes have just been added.
+ //
+ // Every per-disk runtime, not just the first: a vid mounts as one
+ // EcVolume per disk, the delivery lands the .ecsum on one of them, and
+ // the first-match lookup would leave the siblings reporting no
+ // protection. Each re-resolves against its own data and index
+ // directories, so a shared -dir.idx reaches all of them.
+ // Resolving across every EC metadata directory is what makes that
+ // reload mean something: startup mirroring gives each shard-bearing
+ // disk its own .ecx/.ecj/.vif but deliberately not the sidecar, so a
+ // runtime restricted to its own two directories would find nothing
+ // however often it reloaded. One delivered copy, reachable from all.
+ let ec_metadata_dirs = store.ec_metadata_dirs();
+ for ec_vol in store.find_all_ec_volumes_mut(vid) {
+ ec_vol.reload_bitrot_sidecar(&ec_metadata_dirs);
+ }
drop(store);
self.state.volume_state_notify.notify_one();
@@ -3349,6 +3443,8 @@ impl VolumeServer for VolumeGrpcService {
let ecx_dir = ec_vol.ecx_actual_dir().to_string();
let collection = ec_vol.collection.clone();
let vif_dat_file_size = ec_vol.dat_file_size;
+ let (large_block_size, small_block_size) =
+ (ec_vol.large_block_size(), ec_vol.small_block_size());
// shard_dirs[i] is guaranteed Some for i in 0..data_shards by
// the check above; collect concrete dirs for the decoder.
let per_shard_dirs: Vec<String> = shard_dirs[..data_shards]
@@ -3382,6 +3478,8 @@ impl VolumeServer for VolumeGrpcService {
vif_dat_file_size,
data_shards,
&per_shard_dirs,
+ large_block_size as usize,
+ small_block_size as usize,
)
.map_err(|e| Status::internal(format!("WriteDatFile: {}", e)))?;
@@ -3440,12 +3538,24 @@ impl VolumeServer for VolumeGrpcService {
.walk_ecx_stats()
.map_err(|e| Status::internal(e.to_string()))?;
+ // The layout this holder serves reads through, as Go reports it: a
+ // coordinator cannot otherwise tell a holder that understands the
+ // uniform block layout from one that dropped the unknown .vif field
+ // and mounted the volume as legacy.
+ let ec_shard_config = Some(volume_server_pb::EcShardConfig {
+ data_shards: ec_vol.data_shards,
+ parity_shards: ec_vol.parity_shards,
+ encode_ts_ns: 0,
+ block_size: ec_vol.block_size,
+ });
+
Ok(Response::new(
volume_server_pb::VolumeEcShardsInfoResponse {
ec_shard_infos: shard_infos,
volume_size,
file_count,
file_deleted_count,
+ ec_shard_config,
},
))
}
@@ -5086,6 +5196,110 @@ mod tests {
use tempfile::TempDir;
use tokio_stream::StreamExt;
+ fn loc(dir: &str, idx_dir: &str, shard_count: usize, has_ecx: bool) -> LocInfo {
+ LocInfo {
+ dir: dir.to_string(),
+ idx_dir: idx_dir.to_string(),
+ shard_count,
+ has_ecx,
+ }
+ }
+
+ // The rebuild reads its shards from one directory but resolves the volume's
+ // layout -- ratio and uniform block size -- from the .vif or the
+ // generation-0 .ecsum, which on a multi-disk server may sit anywhere. Every
+ // directory that could hold one has to be in the search list, or the
+ // resolution silently falls back to 10+4 with the legacy striping and
+ // reconstructs through the wrong matrix.
+ // The rebuild's own data and index directories are handed to the resolvers
+ // as their own arguments, so they are deliberately absent from this list --
+ // unlike Go, whose resolver takes a single directory list and therefore
+ // carries the rebuild's index directory inside it.
+ #[test]
+ fn select_rebuild_location_excludes_the_rebuilds_own_dirs() {
+ for infos in [
+ vec![loc("/data1", "/idx1", 3, true)],
+ vec![loc("/data1", "/data1", 3, true)],
+ ] {
+ let (idx, others) = select_rebuild_location(&infos).expect("a location with .ecx");
+ assert_eq!(idx, 0);
+ assert!(others.is_empty(), "got {:?}", others);
+ }
+ }
+
+ // The case two reviewers flagged: a sibling holding only shards while its
+ // index directory holds this volume's .vif.
+ #[test]
+ fn select_rebuild_location_searches_a_siblings_index_dir_not_just_its_data_dir() {
+ let infos = vec![
+ loc("/data1", "/data1", 5, true),
+ loc("/data2", "/idx2", 2, false),
+ ];
+ let (idx, others) = select_rebuild_location(&infos).expect("a location with .ecx");
+ assert_eq!(idx, 0);
+ assert_eq!(others, vec!["/data2".to_string(), "/idx2".to_string()]);
+ }
+
+ // Several disks pointed at one index directory is a normal -dir.idx
+ // deployment; the shared directory is worth searching but only once.
+ #[test]
+ fn select_rebuild_location_lists_a_shared_index_dir_once() {
+ let infos = vec![
+ loc("/data1", "/data1", 5, true),
+ loc("/data2", "/shared-idx", 2, false),
+ loc("/data3", "/shared-idx", 1, false),
+ ];
+ let (_, others) = select_rebuild_location(&infos).expect("a location with .ecx");
+ assert_eq!(
+ others,
+ vec![
+ "/data2".to_string(),
+ "/data3".to_string(),
+ "/shared-idx".to_string()
+ ]
+ );
+ }
+
+ // When the shared index directory is the rebuild's own it drops out, since
+ // the caller passes it separately.
+ #[test]
+ fn select_rebuild_location_omits_a_shared_index_dir_it_rebuilds_into() {
+ let infos = vec![
+ loc("/data1", "/shared-idx", 5, true),
+ loc("/data2", "/shared-idx", 2, false),
+ ];
+ let (_, others) = select_rebuild_location(&infos).expect("a location with .ecx");
+ assert_eq!(others, vec!["/data2".to_string()]);
+ }
+
+ // The winner moves as a fuller location turns up; the one it displaces
+ // still has to be searched, index directory included.
+ #[test]
+ fn select_rebuild_location_keeps_the_displaced_winners_dirs() {
+ let infos = vec![
+ loc("/data1", "/idx1", 2, true),
+ loc("/data2", "/idx2", 9, true),
+ ];
+ let (idx, others) = select_rebuild_location(&infos).expect("a location with .ecx");
+ assert_eq!(idx, 1, "the fuller location wins");
+ assert_eq!(others, vec!["/data1".to_string(), "/idx1".to_string()]);
+ }
+
+ #[test]
+ fn select_rebuild_location_drops_empty_dirs() {
+ let infos = vec![loc("/data1", "", 3, true), loc("/data2", "", 1, false)];
+ let (_, others) = select_rebuild_location(&infos).expect("a location with .ecx");
+ assert_eq!(others, vec!["/data2".to_string()]);
+ }
+
+ // Nothing carries an .ecx: there is no index to rebuild the shards against,
+ // so the caller answers with an empty rebuild rather than guessing.
+ #[test]
+ fn select_rebuild_location_is_none_without_an_ecx() {
+ let infos = vec![loc("/data1", "/idx1", 3, false)];
+ assert!(select_rebuild_location(&infos).is_none());
+ }
+
#[test]
fn test_parse_grpc_address_with_explicit_grpc_port() {
// Format: "ip:port.grpcPort" — used by SeaweedFS for source_data_node
diff --git a/seaweed-volume/src/server/store_ec.rs b/seaweed-volume/src/server/store_ec.rs
index 82277c500b..1a8715295e 100644
--- a/seaweed-volume/src/server/store_ec.rs
+++ b/seaweed-volume/src/server/store_ec.rs
@@ -599,7 +599,7 @@ fn read_local_intervals(
) -> Vec<IntervalResult> {
let mut interval_results = Vec::with_capacity(intervals.len());
for interval in intervals {
- let (shard_id, shard_offset) = interval.to_shard_id_and_offset(ecv.data_shards);
+ let (shard_id, shard_offset) = ecv.interval_to_shard_id_and_offset(interval);
let buf_size = interval.size as usize;
let local = ecv.shards.get(shard_id as usize).and_then(|s| s.as_ref());
match local {
diff --git a/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs b/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs
index e1eab27b5a..1dc9b42eea 100644
--- a/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs
+++ b/seaweed-volume/src/storage/erasure_coding/ec_bitrot.rs
@@ -465,6 +465,28 @@ pub fn resolve_status(
}
}
+/// Whether a generation-matching sidecar agrees with the geometry the volume is
+/// mounted with. Both files record the layout the generation was encoded with,
+/// so a disagreement means one of them is wrong and reads through the other
+/// would land at the wrong shard offsets — the caller fails the mount rather
+/// than merely dropping protection. A sidecar that records no EC config has
+/// nothing to contradict.
+pub fn geometry_matches(
+ prot: &EcBitrotProtection,
+ data_shards: usize,
+ parity_shards: usize,
+ block_size: i64,
+) -> bool {
+ match &prot.ec_shard_config {
+ None => true,
+ Some(cfg) => {
+ cfg.data_shards as usize == data_shards
+ && cfg.parity_shards as usize == parity_shards
+ && cfg.block_size == block_size
+ }
+ }
+}
+
/// Returns the [`EcShardChecksums`] entry for a shard id, or `None`.
pub fn shard_checksums(prot: &EcBitrotProtection, shard_id: u32) -> Option<&EcShardChecksums> {
prot.shards.iter().find(|s| s.shard_id == shard_id)
@@ -539,11 +561,12 @@ fn read_full_at(f: &File, buf: &mut [u8], offset: u64) -> io::Result<()> {
/// Builds the `EcShardConfig` proto for the given layout. The bitrot sidecar
/// carries its own top-level encode_uuid, so the nested config leaves it empty.
-pub fn ec_shard_config(data_shards: u32, parity_shards: u32) -> EcShardConfig {
+pub fn ec_shard_config(data_shards: u32, parity_shards: u32, block_size: i64) -> EcShardConfig {
EcShardConfig {
data_shards,
parity_shards,
encode_ts_ns: 0,
+ block_size,
}
}
@@ -567,6 +590,7 @@ mod tests {
data_shards: 10,
parity_shards: 4,
encode_ts_ns: 0,
+ block_size: 0,
}),
shards: vec![
EcShardChecksums {
@@ -712,7 +736,7 @@ mod tests {
algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
block_size: DEFAULT_BITROT_BLOCK_SIZE as u32,
generation: 0,
- ec_shard_config: Some(ec_shard_config(10, 4)),
+ ec_shard_config: Some(ec_shard_config(10, 4, 0)),
shards: vec![EcShardChecksums {
shard_id: 0,
covered_size: covered,
@@ -750,7 +774,7 @@ mod tests {
algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
block_size: DEFAULT_BITROT_BLOCK_SIZE as u32,
generation: 0,
- ec_shard_config: Some(ec_shard_config(10, 4)),
+ ec_shard_config: Some(ec_shard_config(10, 4, 0)),
shards: vec![EcShardChecksums {
shard_id: 0,
covered_size: 5,
@@ -794,7 +818,7 @@ mod tests {
algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
block_size: DEFAULT_BITROT_BLOCK_SIZE as u32,
generation: 0,
- ec_shard_config: Some(ec_shard_config(10, 4)),
+ ec_shard_config: Some(ec_shard_config(10, 4, 0)),
shards,
encode_uuid: vec![0u8; 16],
}
diff --git a/seaweed-volume/src/storage/erasure_coding/ec_decoder.rs b/seaweed-volume/src/storage/erasure_coding/ec_decoder.rs
index 38a7de5617..81db2d602d 100644
--- a/seaweed-volume/src/storage/erasure_coding/ec_decoder.rs
+++ b/seaweed-volume/src/storage/erasure_coding/ec_decoder.rs
@@ -80,6 +80,8 @@ pub fn write_dat_file_from_shards(
dat_file_size: i64,
encoded_dat_file_size: i64,
data_shards: usize,
+ large_block_size: usize,
+ small_block_size: usize,
) -> io::Result<()> {
let dirs: Vec<String> = (0..data_shards).map(|_| dir.to_string()).collect();
write_dat_file_from_shards_with_dirs(
@@ -90,6 +92,8 @@ pub fn write_dat_file_from_shards(
encoded_dat_file_size,
data_shards,
&dirs,
+ large_block_size,
+ small_block_size,
)
}
@@ -113,7 +117,10 @@ pub fn write_dat_file_from_shards(
/// boundary, and deriving the layout from the shrunk extent would read
/// the shards in the wrong block order. Pass zero when the .vif does
/// not record the encode-time size to infer the layout from the shard
-/// size.
+/// size. `large_block_size`/`small_block_size` are the volume's shard
+/// block layout, e.g. `EcVolume::large_block_size()` /
+/// `small_block_size()` from its .vif EC config.
+#[allow(clippy::too_many_arguments)]
pub fn write_dat_file_from_shards_with_dirs(
dat_dir: &str,
collection: &str,
@@ -122,6 +129,8 @@ pub fn write_dat_file_from_shards_with_dirs(
encoded_dat_file_size: i64,
data_shards: usize,
shard_dirs: &[String],
+ large_block_size: usize,
+ small_block_size: usize,
) -> io::Result<()> {
write_dat_file(
dat_dir,
@@ -131,8 +140,8 @@ pub fn write_dat_file_from_shards_with_dirs(
encoded_dat_file_size,
data_shards,
shard_dirs,
- ERASURE_CODING_LARGE_BLOCK_SIZE,
- ERASURE_CODING_SMALL_BLOCK_SIZE,
+ large_block_size,
+ small_block_size,
)
}
@@ -412,7 +421,9 @@ mod tests {
// Encode to EC
let data_shards = 10;
let parity_shards = 4;
- ec_encoder::write_ec_files(dir, dir, "", VolumeId(1), data_shards, parity_shards).unwrap();
+ let block_size =
+ ec_encoder::write_ec_files(dir, dir, "", VolumeId(1), data_shards, parity_shards)
+ .unwrap();
// Delete original .dat and .idx
std::fs::remove_file(format!("{}/1.dat", dir)).unwrap();
@@ -426,6 +437,8 @@ mod tests {
original_dat_size as i64,
original_dat_size as i64,
data_shards,
+ block_size as usize,
+ block_size as usize,
)
.unwrap();
write_idx_file_from_ec_index(dir, "", VolumeId(1)).unwrap();
@@ -472,7 +485,16 @@ mod tests {
let dir = tmp.path().to_str().unwrap();
// No shard files exist, so de-striping must fail and publish nothing:
// neither the final .dat nor a partial .dat.tmp may remain.
- let res = write_dat_file_from_shards(dir, "", VolumeId(7), 100, 100, 10);
+ let res = write_dat_file_from_shards(
+ dir,
+ "",
+ VolumeId(7),
+ 100,
+ 100,
+ 10,
+ ERASURE_CODING_LARGE_BLOCK_SIZE,
+ ERASURE_CODING_SMALL_BLOCK_SIZE,
+ );
assert!(res.is_err());
assert!(!std::path::Path::new(&format!("{}/7.dat", dir)).exists());
assert!(!std::path::Path::new(&format!("{}/7.dat.tmp", dir)).exists());
@@ -525,6 +547,7 @@ mod tests {
&mut builders,
data_shards,
parity_shards,
+ SMALL,
LARGE,
SMALL,
)
@@ -616,6 +639,7 @@ mod tests {
&mut builders,
data_shards,
parity_shards,
+ SMALL,
LARGE,
SMALL,
)
diff --git a/seaweed-volume/src/storage/erasure_coding/ec_encoder.rs b/seaweed-volume/src/storage/erasure_coding/ec_encoder.rs
index 346b2fd1f5..1470e85de3 100644
--- a/seaweed-volume/src/storage/erasure_coding/ec_encoder.rs
+++ b/seaweed-volume/src/storage/erasure_coding/ec_encoder.rs
@@ -25,6 +25,10 @@ use crate::storage::volume::volume_file_name;
///
/// Creates .ec00-.ec13 files in the same directory.
/// Also creates a sorted .ecx index from the .idx file.
+///
+/// Always encodes with the uniform block layout, sized for this .dat, and
+/// returns the block size so the caller can persist it to .vif. Mirrors Go's
+/// WriteEcFiles.
pub fn write_ec_files(
dir: &str,
idx_dir: &str,
@@ -32,7 +36,7 @@ pub fn write_ec_files(
volume_id: VolumeId,
data_shards: usize,
parity_shards: usize,
-) -> io::Result<()> {
+) -> io::Result<i64> {
let base = volume_file_name(dir, collection, volume_id);
let dat_path = format!("{}.dat", base);
let idx_base = volume_file_name(idx_dir, collection, volume_id);
@@ -66,7 +70,7 @@ pub fn write_ec_files(
.map(|_| ShardChecksumBuilder::new(DEFAULT_BITROT_BLOCK_SIZE as i64))
.collect();
- // Encode in large blocks, then small blocks
+ let block_size = uniform_block_size(dat_size, data_shards);
encode_dat_file(
&dat_file,
dat_size,
@@ -75,8 +79,9 @@ pub fn write_ec_files(
&mut builders,
data_shards,
parity_shards,
- ERASURE_CODING_LARGE_BLOCK_SIZE,
- ERASURE_CODING_SMALL_BLOCK_SIZE,
+ ENCODE_BUFFER_SIZE,
+ block_size as usize,
+ block_size as usize,
)?;
// Close all shards
@@ -103,6 +108,7 @@ pub fn write_ec_files(
ec_shard_config: Some(ec_bitrot::ec_shard_config(
data_shards as u32,
parity_shards as u32,
+ block_size,
)),
shards: shard_checksums,
encode_uuid: ec_bitrot::new_encode_uuid(),
@@ -120,7 +126,19 @@ pub fn write_ec_files(
);
}
- Ok(())
+ Ok(block_size)
+}
+
+/// uniform_block_size returns the per-shard block size of the uniform layout
+/// for a .dat of the given size: ceil(dat_file_size/data_shards) rounded up to
+/// a whole small block. For every input this equals the legacy layout's padded
+/// shard size, so only the byte placement differs between the two layouts,
+/// never the shard length. Mirrors Go's UniformBlockSize.
+pub fn uniform_block_size(dat_file_size: i64, data_shards: usize) -> i64 {
+ let small = ERASURE_CODING_SMALL_BLOCK_SIZE as i64;
+ let per_shard = (dat_file_size + data_shards as i64 - 1) / data_shards as i64;
+ let blocks = ((per_shard + small - 1) / small).max(1);
+ blocks * small
}
/// Rebuild missing EC shard files from existing shards using Reed-Solomon reconstruct.
@@ -370,7 +388,7 @@ pub fn verify_ec_shards(
}
/// Write sorted .ecx index from .idx file.
-fn write_sorted_ecx_from_idx(idx_path: &str, ecx_path: &str) -> io::Result<()> {
+pub(crate) fn write_sorted_ecx_from_idx(idx_path: &str, ecx_path: &str) -> io::Result<()> {
if !std::path::Path::new(idx_path).exists() {
return Err(io::Error::new(
io::ErrorKind::NotFound,
@@ -421,6 +439,8 @@ pub fn rebuild_ecx_file(
collection: &str,
volume_id: VolumeId,
data_shards: usize,
+ block_size: i64,
+ dat_file_size: i64,
additional_dirs: &[&str],
) -> io::Result<()> {
use crate::storage::needle::needle::get_actual_size;
@@ -463,10 +483,42 @@ pub fn rebuild_ecx_file(
// Determine total logical data size from shard sizes
let shard_size = shards.iter().map(|s| s.file_size()).max().unwrap_or(0);
let total_data_size = shard_size as i64 * data_shards as i64;
+ // The volume's shard block layout: the .vif-recorded uniform block size,
+ // or the legacy two-tier sizes when 0. The row count comes from the shard
+ // length; -1 disambiguates a legacy shard that is an exact large-block
+ // multiple (mirrors the ecdFileSize-1 fallback in the read path).
+ let (large_block, small_block) = if block_size > 0 {
+ (block_size, block_size)
+ } else {
+ (
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64,
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ )
+ };
+ // The row count the de-stripe walks with. The encode-time .dat size is the
+ // authority — the same value the read path divides by data_shards — and the
+ // padded extent is only a fallback: under the legacy layout a shard that is
+ // an exact large-block multiple reads as one row too many, which
+ // re-interprets its last large row as small blocks and scrambles the
+ // recovered offsets. Subtracting one keeps that fallback on the safe side of
+ // the boundary, exactly as the read path's own fallback does.
+ let locate_shard_size = if dat_file_size > 0 {
+ dat_file_size / data_shards as i64
+ } else {
+ (shard_size as i64 - 1).max(0)
+ };
// Read version from superblock (first byte of logical data)
let mut sb_buf = [0u8; SUPER_BLOCK_SIZE];
- read_from_data_shards(&shards, &mut sb_buf, 0, data_shards)?;
+ read_from_data_shards(
+ &shards,
+ &mut sb_buf,
+ 0,
+ data_shards,
+ locate_shard_size,
+ large_block,
+ small_block,
+ )?;
let version = Version(sb_buf[0]);
// Walk needles starting after superblock
@@ -475,10 +527,30 @@ pub fn rebuild_ecx_file(
let mut entries: Vec<(NeedleId, Offset, Size)> = Vec::new();
while offset + header_size as i64 <= total_data_size {
- // Read needle header (cookie + needle_id + size = 16 bytes)
+ // Read needle header (cookie + needle_id + size = 16 bytes).
+ // A read failure is NOT the end of the data — every offset in
+ // range maps into the shards, so an error means a truncated or
+ // unreadable shard. Publishing the entries collected so far as
+ // a successful .ecx would hand out a silently incomplete
+ // recovery index; propagate instead. (The scan still ends
+ // normally on the zero-cookie tail below.)
let mut header_buf = [0u8; NEEDLE_HEADER_SIZE];
- if read_from_data_shards(&shards, &mut header_buf, offset as u64, data_shards).is_err() {
- break;
+ if let Err(e) = read_from_data_shards(
+ &shards,
+ &mut header_buf,
+ offset as u64,
+ data_shards,
+ locate_shard_size,
+ large_block,
+ small_block,
+ ) {
+ for s in &mut shards {
+ s.close();
+ }
+ return Err(io::Error::new(
+ e.kind(),
+ format!("scan needle header at offset {}: {}", offset, e),
+ ));
}
let cookie = Cookie::from_bytes(&header_buf[..COOKIE_SIZE]);
@@ -532,58 +604,83 @@ pub fn rebuild_ecx_file(
Ok(())
}
-/// Read bytes from EC data shards at a logical offset in the .dat file.
+/// Read bytes from EC data shards at a logical offset in the .dat file,
+/// resolving the shard/offset through the volume's block layout via
+/// locate_data — the same mapping the read path uses.
+#[allow(clippy::too_many_arguments)]
fn read_from_data_shards(
shards: &[EcVolumeShard],
buf: &mut [u8],
logical_offset: u64,
data_shards: usize,
+ locate_shard_size: i64,
+ large_block_size: i64,
+ small_block_size: i64,
) -> io::Result<()> {
- let small_block = ERASURE_CODING_SMALL_BLOCK_SIZE as u64;
- let data_shards_u64 = data_shards as u64;
-
- let mut bytes_read = 0u64;
- let mut remaining = buf.len() as u64;
- let mut current_offset = logical_offset;
-
- while remaining > 0 {
- // Determine which shard and at what shard-offset this logical offset maps to.
- // The data is interleaved: large blocks first, then small blocks.
- // For simplicity, use the small block size for all calculations since
- // large blocks are multiples of small blocks.
- let row_size = small_block * data_shards_u64;
- let row_index = current_offset / row_size;
- let row_offset = current_offset % row_size;
- let shard_index = (row_offset / small_block) as usize;
- let shard_offset = row_index * small_block + (row_offset % small_block);
-
- if shard_index >= data_shards {
+ let intervals = crate::storage::erasure_coding::ec_locate::locate_data(
+ logical_offset as i64,
+ Size(buf.len() as i32),
+ locate_shard_size,
+ data_shards as u32,
+ large_block_size,
+ small_block_size,
+ );
+ let mut bytes_read = 0usize;
+ for interval in &intervals {
+ let (shard_id, shard_offset) =
+ interval.to_shard_id_and_offset(data_shards as u32, large_block_size, small_block_size);
+ if shard_id as usize >= data_shards {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"shard index out of range",
));
}
-
- // How many bytes can we read from this position in this shard block
- let bytes_left_in_block = small_block - (row_offset % small_block);
- let to_read = remaining.min(bytes_left_in_block) as usize;
-
- let dest = &mut buf[bytes_read as usize..bytes_read as usize + to_read];
- shards[shard_index].read_at(dest, shard_offset)?;
-
- bytes_read += to_read as u64;
- remaining -= to_read as u64;
- current_offset += to_read as u64;
+ let to_read = interval.size as usize;
+ let dest = &mut buf[bytes_read..bytes_read + to_read];
+ // Exact-read semantics: read_at may legally return fewer bytes
+ // than requested, and treating a short read as complete leaves
+ // the tail of `dest` as whatever the buffer held before. Loop
+ // until filled; zero bytes inside the mapped range means the
+ // shard is truncated — an error, not an end.
+ let mut filled = 0usize;
+ while filled < to_read {
+ let n = shards[shard_id as usize]
+ .read_at(&mut dest[filled..], shard_offset as u64 + filled as u64)?;
+ if n == 0 {
+ return Err(io::Error::new(
+ io::ErrorKind::UnexpectedEof,
+ format!(
+ "short read from data shard {}: {} of {} bytes at offset {}",
+ shard_id, filled, to_read, shard_offset
+ ),
+ ));
+ }
+ filled += n;
+ }
+ bytes_read += to_read;
+ }
+ if bytes_read != buf.len() {
+ return Err(io::Error::new(
+ io::ErrorKind::UnexpectedEof,
+ "short read from data shards",
+ ));
}
-
Ok(())
}
+/// Buffer size for one encode sub-batch per shard, mirroring Go's 256KB
+/// bufferSize in WriteEcFiles. A block is processed in block_size/buffer_size
+/// sub-batches, so memory stays at total_shards * 256KB no matter how large
+/// the uniform block is.
+const ENCODE_BUFFER_SIZE: usize = 256 * 1024;
+
/// Encode the .dat file data into shard files.
///
/// Uses a two-phase approach matching Go's ec_encoder.go:
-/// 1. Process as many large blocks (1GB) as possible
-/// 2. Process remaining data with small blocks (1MB)
+/// 1. Process as many large blocks as possible
+/// 2. Process remaining data with small blocks
+///
+/// `buffer_size` must divide both block sizes.
#[allow(clippy::too_many_arguments)]
pub(crate) fn encode_dat_file(
dat_file: &File,
@@ -593,44 +690,50 @@ pub(crate) fn encode_dat_file(
builders: &mut [ShardChecksumBuilder],
data_shards: usize,
parity_shards: usize,
+ buffer_size: usize,
large_block_size: usize,
small_block_size: usize,
) -> io::Result<()> {
+ let total_shards = data_shards + parity_shards;
+ let mut buffers: Vec<Vec<u8>> = (0..total_shards)
+ .map(|_| vec![0u8; buffer_size])
+ .collect();
+
let mut remaining = dat_size;
let mut offset: u64 = 0;
- // Phase 1: Process large blocks (1GB each) while enough data remains
+ // Phase 1: process whole large-block rows while enough data remains
let large_row_size = large_block_size * data_shards;
while remaining >= large_row_size as i64 {
- encode_one_batch(
+ encode_data(
dat_file,
offset,
large_block_size,
rs,
+ &mut buffers,
shards,
builders,
data_shards,
- parity_shards,
)?;
offset += large_row_size as u64;
remaining -= large_row_size as i64;
}
- // Phase 2: Process remaining data with small blocks (1MB each)
+ // Phase 2: process remaining data with small blocks
let small_row_size = small_block_size * data_shards;
while remaining > 0 {
let to_process = remaining.min(small_row_size as i64);
- encode_one_batch(
+ encode_data(
dat_file,
offset,
small_block_size,
rs,
+ &mut buffers,
shards,
builders,
data_shards,
- parity_shards,
)?;
offset += to_process as u64;
remaining -= to_process;
@@ -639,61 +742,71 @@ pub(crate) fn encode_dat_file(
Ok(())
}
-/// Encode one batch (row) of data.
-fn encode_one_batch(
+/// Encode one row of blocks, streaming it in ENCODE_BUFFER_SIZE sub-batches so
+/// arbitrarily large blocks never require block-sized allocations. Mirrors
+/// Go's encodeData.
+#[allow(clippy::too_many_arguments)]
+fn encode_data(
dat_file: &File,
- offset: u64,
+ row_offset: u64,
block_size: usize,
rs: &ReedSolomon,
+ buffers: &mut [Vec<u8>],
shards: &mut [EcVolumeShard],
builders: &mut [ShardChecksumBuilder],
data_shards: usize,
- parity_shards: usize,
) -> io::Result<()> {
- let total_shards = data_shards + parity_shards;
- // Each batch allocates block_size * total_shards bytes.
- // With large blocks (1 GiB) this is 14 GiB -- guard against OOM.
- let total_alloc = block_size.checked_mul(total_shards).ok_or_else(|| {
- io::Error::new(
- io::ErrorKind::InvalidInput,
- "block_size * shard count overflows usize",
- )
- })?;
- // Large-block encoding uses 1 GiB * 14 shards = 14 GiB; allow up to 16 GiB.
- const MAX_BATCH_ALLOC: usize = 16 * 1024 * 1024 * 1024; // 16 GiB safety limit
- if total_alloc > MAX_BATCH_ALLOC {
+ let buffer_size = buffers[0].len();
+ if block_size % buffer_size != 0 {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!(
- "batch allocation too large ({} bytes, limit {} bytes); block_size={} shards={}",
- total_alloc, MAX_BATCH_ALLOC, block_size, total_shards,
+ "unexpected block size {} buffer size {}",
+ block_size, buffer_size
),
));
}
+ let batch_count = block_size / buffer_size;
+ for b in 0..batch_count {
+ encode_one_batch(
+ dat_file,
+ row_offset + (b * buffer_size) as u64,
+ block_size,
+ rs,
+ buffers,
+ shards,
+ builders,
+ data_shards,
+ )?;
+ }
+ Ok(())
+}
- // Allocate buffers for all shards
- let mut buffers: Vec<Vec<u8>> = (0..total_shards).map(|_| vec![0u8; block_size]).collect();
-
- // Read data shards from .dat file
+/// Encode one sub-batch: the same buffer-sized slice of every shard's block in
+/// this row. Mirrors Go's encodeDataOneBatch.
+#[allow(clippy::too_many_arguments)]
+fn encode_one_batch(
+ dat_file: &File,
+ offset: u64,
+ block_size: usize,
+ rs: &ReedSolomon,
+ buffers: &mut [Vec<u8>],
+ shards: &mut [EcVolumeShard],
+ builders: &mut [ShardChecksumBuilder],
+ data_shards: usize,
+) -> io::Result<()> {
+ // Read data shards from the .dat file, zero-filling past EOF — the buffers
+ // are reused across batches, so the tail must be cleared explicitly.
for i in 0..data_shards {
let read_offset = offset + (i * block_size) as u64;
-
- #[cfg(unix)]
- {
- use std::os::unix::fs::FileExt;
- dat_file.read_at(&mut buffers[i], read_offset)?;
- }
-
- #[cfg(not(unix))]
- {
- let mut f = dat_file.try_clone()?;
- f.seek(SeekFrom::Start(read_offset))?;
- f.read(&mut buffers[i])?;
+ let n = read_at_most(dat_file, &mut buffers[i], read_offset)?;
+ for b in buffers[i][n..].iter_mut() {
+ *b = 0;
}
}
// Encode parity shards
- rs.encode(&mut buffers).map_err(|e| {
+ rs.encode(&mut *buffers).map_err(|e| {
io::Error::new(
io::ErrorKind::Other,
format!("reed-solomon encode: {:?}", e),
@@ -710,6 +823,29 @@ fn encode_one_batch(
Ok(())
}
+/// Read into `buf` at `offset` until it is full or EOF; returns bytes read.
+fn read_at_most(dat_file: &File, buf: &mut [u8], offset: u64) -> io::Result<usize> {
+ let mut n = 0;
+ while n < buf.len() {
+ #[cfg(unix)]
+ let r = {
+ use std::os::unix::fs::FileExt;
+ dat_file.read_at(&mut buf[n..], offset + n as u64)?
+ };
+ #[cfg(not(unix))]
+ let r = {
+ let mut f = dat_file.try_clone()?;
+ f.seek(SeekFrom::Start(offset + n as u64))?;
+ f.read(&mut buf[n..])?
+ };
+ if r == 0 {
+ break;
+ }
+ n += r;
+ }
+ Ok(n)
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -1020,7 +1156,7 @@ mod tests {
// Without additional_dirs, rebuild must fail: shards 1, 3, 6 are not
// in primary and the full logical .dat content can't be reconstructed.
- let res = rebuild_ecx_file(&primary, "", VolumeId(1), 10, &[]);
+ let res = rebuild_ecx_file(&primary, "", VolumeId(1), 10, 0, 0, &[]);
assert!(
res.is_err(),
"ecx rebuild without additional_dirs must fail when data shards are on another disk"
@@ -1031,7 +1167,7 @@ mod tests {
);
// With additional_dirs pointing at the secondary, the rebuild must succeed.
- rebuild_ecx_file(&primary, "", VolumeId(1), 10, &[secondary.as_str()]).unwrap();
+ rebuild_ecx_file(&primary, "", VolumeId(1), 10, 0, 0, &[secondary.as_str()]).unwrap();
assert!(
std::path::Path::new(&ecx_path).exists(),
@@ -1043,6 +1179,118 @@ mod tests {
);
}
+ // A uniform-layout volume (block size > 1MiB) must have its .ecx rebuilt
+ // through the recorded geometry; the legacy 1MiB mapping would scan
+ // garbage past the first block boundary.
+ #[test]
+ fn test_rebuild_ecx_file_uniform_layout() {
+ use crate::storage::needle_map::NeedleMapKind;
+ use crate::storage::volume::Volume;
+ let tmp = TempDir::new().unwrap();
+ let dir = tmp.path().to_str().unwrap().to_string();
+ let mut v = Volume::new(
+ &dir,
+ &dir,
+ "",
+ VolumeId(2),
+ NeedleMapKind::InMemory,
+ None,
+ None,
+ 0,
+ Version::current(),
+ )
+ .unwrap();
+ for i in 1u64..=12 {
+ let data: Vec<u8> = (0..2 << 20)
+ .map(|b| ((b as u64).wrapping_mul(2654435761).wrapping_add(i) >> 8) as u8)
+ .collect();
+ let mut n = Needle {
+ id: NeedleId(i),
+ cookie: Cookie(i as u32),
+ data: data.clone(),
+ data_size: data.len() as u32,
+ ..Needle::default()
+ };
+ v.write_needle(&mut n, true, false).unwrap();
+ }
+ v.sync_to_disk().unwrap();
+ v.close();
+ let block_size = write_ec_files(&dir, &dir, "", VolumeId(2), 10, 4).unwrap();
+ assert!(
+ block_size > ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ "fixture must diverge from the legacy layout"
+ );
+
+ let ecx_path = format!("{}/2.ecx", dir);
+ let canonical = std::fs::read(&ecx_path).unwrap();
+ std::fs::remove_file(&ecx_path).unwrap();
+
+ rebuild_ecx_file(&dir, "", VolumeId(2), 10, block_size, 0, &[]).unwrap();
+ let rebuilt = std::fs::read(&ecx_path).unwrap();
+ assert_eq!(canonical, rebuilt, "rebuilt .ecx must match the encode-time .ecx");
+ }
+
+ // A truncated data shard must FAIL the .ecx rebuild, not publish the
+ // entries scanned so far as a successful (silently incomplete) index.
+ #[test]
+ fn test_rebuild_ecx_file_fails_on_truncated_shard() {
+ use crate::storage::needle_map::NeedleMapKind;
+ use crate::storage::volume::Volume;
+ let tmp = TempDir::new().unwrap();
+ let dir = tmp.path().to_str().unwrap().to_string();
+ let mut v = Volume::new(
+ &dir,
+ &dir,
+ "",
+ VolumeId(3),
+ NeedleMapKind::InMemory,
+ None,
+ None,
+ 0,
+ Version::current(),
+ )
+ .unwrap();
+ for i in 1u64..=12 {
+ let data: Vec<u8> = (0..2 << 20)
+ .map(|b| ((b as u64).wrapping_mul(2654435761).wrapping_add(i) >> 8) as u8)
+ .collect();
+ let mut n = Needle {
+ id: NeedleId(i),
+ cookie: Cookie(i as u32),
+ data: data.clone(),
+ data_size: data.len() as u32,
+ ..Needle::default()
+ };
+ v.write_needle(&mut n, true, false).unwrap();
+ }
+ v.sync_to_disk().unwrap();
+ v.close();
+ let block_size = write_ec_files(&dir, &dir, "", VolumeId(3), 10, 4).unwrap();
+
+ let ecx_path = format!("{}/3.ecx", dir);
+ std::fs::remove_file(&ecx_path).unwrap();
+
+ // Truncate shard 0 to just the superblock: the scan's very first
+ // needle-header read (offset SUPER_BLOCK_SIZE, shard 0 under the
+ // uniform layout) lands in the missing region. The pre-fix code
+ // broke the scan there and published an EMPTY .ecx as success.
+ let shard_path = format!("{}/3.ec00", dir);
+ let f = std::fs::OpenOptions::new()
+ .write(true)
+ .open(&shard_path)
+ .unwrap();
+ f.set_len(crate::storage::super_block::SUPER_BLOCK_SIZE as u64)
+ .unwrap();
+ drop(f);
+
+ let res = rebuild_ecx_file(&dir, "", VolumeId(3), 10, block_size, 0, &[]);
+ assert!(res.is_err(), "rebuild over a truncated shard must fail");
+ assert!(
+ !std::path::Path::new(&ecx_path).exists(),
+ "a failed rebuild must not leave a partial .ecx behind"
+ );
+ }
+
#[test]
fn test_reed_solomon_basic() {
let data_shards = 10;
diff --git a/seaweed-volume/src/storage/erasure_coding/ec_locate.rs b/seaweed-volume/src/storage/erasure_coding/ec_locate.rs
index 4c1f06aa2d..baaed9f3b7 100644
--- a/seaweed-volume/src/storage/erasure_coding/ec_locate.rs
+++ b/seaweed-volume/src/storage/erasure_coding/ec_locate.rs
@@ -18,21 +18,26 @@ pub struct Interval {
}
impl Interval {
- pub fn to_shard_id_and_offset(&self, data_shards: u32) -> (ShardId, i64) {
+ pub fn to_shard_id_and_offset(
+ &self,
+ data_shards: u32,
+ large_block_size: i64,
+ small_block_size: i64,
+ ) -> (ShardId, i64) {
let data_shards_usize = data_shards as usize;
let shard_id = (self.block_index % data_shards_usize) as ShardId;
let row_index = self.block_index / data_shards_usize;
let block_size = if self.is_large_block {
- ERASURE_CODING_LARGE_BLOCK_SIZE as i64
+ large_block_size
} else {
- ERASURE_CODING_SMALL_BLOCK_SIZE as i64
+ small_block_size
};
let mut offset = row_index as i64 * block_size + self.inner_block_offset;
if !self.is_large_block {
// Small blocks come after large blocks in the shard file
- offset += self.large_block_rows_count as i64 * ERASURE_CODING_LARGE_BLOCK_SIZE as i64;
+ offset += self.large_block_rows_count as i64 * large_block_size;
}
(shard_id, offset)
@@ -42,7 +47,14 @@ impl Interval {
/// Locate the EC shard intervals needed to read data at the given offset and size.
///
/// `shard_size` is the size of a single shard file.
-pub fn locate_data(offset: i64, size: Size, shard_size: i64, data_shards: u32) -> Vec<Interval> {
+pub fn locate_data(
+ offset: i64,
+ size: Size,
+ shard_size: i64,
+ data_shards: u32,
+ large_block_size: i64,
+ small_block_size: i64,
+) -> Vec<Interval> {
let mut intervals = Vec::new();
let data_size = size.0 as i64;
@@ -50,17 +62,14 @@ pub fn locate_data(offset: i64, size: Size, shard_size: i64, data_shards: u32) -
return intervals;
}
- let large_block_size = ERASURE_CODING_LARGE_BLOCK_SIZE as i64;
- let small_block_size = ERASURE_CODING_SMALL_BLOCK_SIZE as i64;
let large_row_size = large_block_size * data_shards as i64;
let small_row_size = small_block_size * data_shards as i64;
- // Number of large block rows
- let n_large_block_rows = if shard_size > 0 {
- ((shard_size - 1) / large_block_size) as usize
- } else {
- 0
- };
+ // Number of large block rows. Mirrors Go's shardDatSize/largeBlockLength:
+ // the caller's ecd-size fallback already subtracts 1 to disambiguate the
+ // exact-multiple case, so no further -1 here — a shard size that IS an
+ // exact multiple (dat_file_size path) means real full large rows.
+ let n_large_block_rows = (shard_size / large_block_size) as usize;
let large_section_size = n_large_block_rows as i64 * large_row_size;
let mut remaining_offset = offset;
@@ -150,7 +159,11 @@ mod tests {
is_large_block: true,
large_block_rows_count: 1,
};
- let (shard_id, offset) = interval.to_shard_id_and_offset(data_shards);
+ let (shard_id, offset) = interval.to_shard_id_and_offset(
+ data_shards,
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64,
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ );
assert_eq!(shard_id, 0);
assert_eq!(offset, 100);
@@ -162,7 +175,11 @@ mod tests {
is_large_block: true,
large_block_rows_count: 1,
};
- let (shard_id, _offset) = interval.to_shard_id_and_offset(data_shards);
+ let (shard_id, _offset) = interval.to_shard_id_and_offset(
+ data_shards,
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64,
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ );
assert_eq!(shard_id, 5);
// Block index 12 (data_shards=10) → row_index 1, shard_id 2
@@ -173,7 +190,11 @@ mod tests {
is_large_block: true,
large_block_rows_count: 5,
};
- let (shard_id, offset) = interval.to_shard_id_and_offset(data_shards);
+ let (shard_id, offset) = interval.to_shard_id_and_offset(
+ data_shards,
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64,
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ );
assert_eq!(shard_id, 2); // 12 % 10 = 2
assert_eq!(offset, large_block_size + 200); // row 1 offset + inner_block_offset
@@ -185,7 +206,11 @@ mod tests {
is_large_block: true,
large_block_rows_count: 2,
};
- let (shard_id, offset) = interval.to_shard_id_and_offset(data_shards);
+ let (shard_id, offset) = interval.to_shard_id_and_offset(
+ data_shards,
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64,
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ );
assert_eq!(shard_id, 0);
assert_eq!(offset, ERASURE_CODING_LARGE_BLOCK_SIZE as i64); // row 1 offset
}
@@ -193,7 +218,14 @@ mod tests {
#[test]
fn test_locate_data_small_file() {
// Small file: 100 bytes at offset 50, shard size = 1MB
- let intervals = locate_data(50, Size(100), 1024 * 1024, 10);
+ let intervals = locate_data(
+ 50,
+ Size(100),
+ 1024 * 1024,
+ 10,
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64,
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ );
assert!(!intervals.is_empty());
// Should be a single small block interval (no large block rows for 1MB shard)
@@ -203,7 +235,14 @@ mod tests {
#[test]
fn test_locate_data_empty() {
- let intervals = locate_data(0, Size(0), 1024 * 1024, 10);
+ let intervals = locate_data(
+ 0,
+ Size(0),
+ 1024 * 1024,
+ 10,
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64,
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ );
assert!(intervals.is_empty());
}
@@ -216,7 +255,11 @@ mod tests {
is_large_block: false,
large_block_rows_count: 2,
};
- let (_shard_id, offset) = interval.to_shard_id_and_offset(10);
+ let (_shard_id, offset) = interval.to_shard_id_and_offset(
+ 10,
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64,
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ );
// Should be after 2 large block rows
assert_eq!(offset, 2 * ERASURE_CODING_LARGE_BLOCK_SIZE as i64);
}
diff --git a/seaweed-volume/src/storage/erasure_coding/ec_volume.rs b/seaweed-volume/src/storage/erasure_coding/ec_volume.rs
index 709d8f485b..34c12aaa9b 100644
--- a/seaweed-volume/src/storage/erasure_coding/ec_volume.rs
+++ b/seaweed-volume/src/storage/erasure_coding/ec_volume.rs
@@ -26,6 +26,9 @@ pub struct EcVolume {
pub dat_file_size: i64,
pub data_shards: u32,
pub parity_shards: u32,
+ /// Uniform block layout: each shard is one contiguous block of this many
+ /// bytes. 0 = legacy 1GiB/1MiB two-tier layout. Loaded from .vif.
+ pub block_size: i64,
ecx_file: Option<File>,
ecx_file_size: i64,
ecj_file: Option<File>,
@@ -101,34 +104,249 @@ fn locate_vif_path(dir: &str, dir_idx: &str, collection: &str, volume_id: Volume
data_vif
}
-/// Read EC data/parity shard counts from `.vif`, defaulting to the
-/// build's standard ratio when no `.vif` is present or is malformed.
+/// Load the volume's `.vif` (data dir first, then idx dir, then the active
+/// generation's versioned sidecar — see [`locate_vif_path`]). Absent
+/// everywhere is legal — legacy volumes predate the sidecar — and returns
+/// `None`; so does a zero-byte stub (an ec.decode copy from a source
+/// without one), mirroring Go's MaybeLoadVolumeInfo. A
+/// present-but-unreadable or malformed `.vif` is an ERROR: silently
+/// defaulting would mount a uniform-layout volume with legacy offset math
+/// and serve wrong bytes with a straight face.
+pub fn load_vif_info(
+ dir: &str,
+ dir_idx: &str,
+ collection: &str,
+ volume_id: VolumeId,
+) -> io::Result<Option<crate::storage::volume::VifVolumeInfo>> {
+ let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
+ match std::fs::read_to_string(&vif_path) {
+ Ok(content) if content.trim().is_empty() => Ok(None),
+ Ok(content) => serde_json::from_str(&content).map(Some).map_err(|e| {
+ io::Error::new(
+ io::ErrorKind::InvalidData,
+ format!("parse {}: {}", vif_path, e),
+ )
+ }),
+ Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
+ Err(e) => Err(io::Error::new(
+ e.kind(),
+ format!("read {}: {}", vif_path, e),
+ )),
+ }
+}
+
+/// Read the EC data/parity shard counts and shard block size from `.vif`,
+/// defaulting to the build's standard ratio and the legacy block layout when
+/// no `.vif` is present. A present-but-unreadable or malformed `.vif`
+/// fails instead of defaulting (see [`load_vif_info`]).
/// Looks at the data dir first, then the idx dir — see [`locate_vif_path`].
pub fn read_ec_shard_config(
dir: &str,
dir_idx: &str,
collection: &str,
volume_id: VolumeId,
-) -> (u32, u32) {
- let mut data_shards = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
- let mut parity_shards = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
- let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
- if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
- if let Ok(vif_info) =
- serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
- {
- if let Some(ec) = vif_info.ec_shard_config {
- if ec.data_shards > 0
- && ec.parity_shards > 0
- && (ec.data_shards + ec.parity_shards) <= MAX_SHARD_COUNT as u32
- {
- data_shards = ec.data_shards;
- parity_shards = ec.parity_shards;
- }
+) -> io::Result<(u32, u32, i64)> {
+ let vif = load_vif_info(dir, dir_idx, collection, volume_id)?;
+ ec_shard_config_from(vif.as_ref(), &[dir, dir_idx], collection, volume_id)
+}
+
+/// Load the volume's `.vif` from the selected location or, failing that, from
+/// any sibling disk — returning the directory it came from so the caller can
+/// resolve the rest of the volume's metadata against the same place. A rebuild
+/// picks one disk to write into, but a multi-disk server may hold that
+/// volume's metadata on another.
+pub fn load_vif_info_across_dirs(
+ dir: &str,
+ dir_idx: &str,
+ other_dirs: &[String],
+ collection: &str,
+ volume_id: VolumeId,
+) -> io::Result<Option<(crate::storage::volume::VifVolumeInfo, String)>> {
+ if let Some(vif) = load_vif_info(dir, dir_idx, collection, volume_id)? {
+ // Report where it actually came from. load_vif_info probes the data
+ // directory then the index directory, so naming `dir` unconditionally
+ // pointed callers at the wrong disk whenever the vif lived with the
+ // index — and a caller resolving the rest of the volume's metadata
+ // against that answer would look in a directory holding none of it.
+ let data_vif = format!(
+ "{}.vif",
+ crate::storage::volume::volume_file_name(dir, collection, volume_id)
+ );
+ let found_in = if std::path::Path::new(&data_vif).exists() {
+ dir.to_string()
+ } else {
+ dir_idx.to_string()
+ };
+ return Ok(Some((vif, found_in)));
+ }
+ for other in other_dirs {
+ if let Some(vif) = load_vif_info(other, other, collection, volume_id)? {
+ return Ok(Some((vif, other.clone())));
+ }
+ }
+ Ok(None)
+}
+
+/// [`read_ec_shard_config`] for a caller that may find the volume's metadata on
+/// any of the server's disks. Without this a rebuild that lands on a disk
+/// holding only shards reads the default 10+4 and the legacy block layout,
+/// then reconstructs a custom-ratio or uniform volume through the wrong matrix
+/// and de-striping geometry.
+pub fn read_ec_shard_config_across_dirs(
+ dir: &str,
+ dir_idx: &str,
+ other_dirs: &[String],
+ collection: &str,
+ volume_id: VolumeId,
+) -> io::Result<(u32, u32, i64)> {
+ // Every directory this volume's metadata could be in. A split -dir/-dir.idx
+ // location keeps .vif and .ecsum with the INDEX, and callers exclude their
+ // own index directory from other_dirs on the understanding that it is
+ // passed here separately — so it is chained explicitly, exactly as the .vif
+ // lookup and Go's findBitrotSidecar both do.
+ let mut candidates: Vec<&str> = Vec::with_capacity(2 + other_dirs.len());
+ candidates.push(dir);
+ if !dir_idx.is_empty() && dir_idx != dir {
+ candidates.push(dir_idx);
+ }
+ candidates.extend(other_dirs.iter().map(|s| s.as_str()));
+
+ // A vif that carries no ecShardConfig answers nothing about the layout, so
+ // it must NOT short-circuit the sidecar search: a legacy config-free vif
+ // and the generation-0 sidecar can sit in different directories, and
+ // stopping at the vif resolved a 12+4 uniform volume as 10+4 legacy. Pass
+ // the whole candidate list so the fallback covers the same ground the vif
+ // lookup did.
+ let vif = load_vif_info_across_dirs(dir, dir_idx, other_dirs, collection, volume_id)?;
+ ec_shard_config_from(
+ vif.as_ref().map(|(v, _)| v),
+ &candidates,
+ collection,
+ volume_id,
+ )
+}
+
+/// Resolve (data_shards, parity_shards, block_size) from an already-loaded
+/// `.vif`. With no vif — or one carrying no EC config — the bitrot sidecar
+/// records the same config at encode time and answers the layout question the
+/// vif cannot: defaulting a uniform-layout volume to the legacy block sizes
+/// maps every read to the wrong shard offset. `weed fix -ecx` reads the
+/// sidecar for the same reason. Absent both, the build's standard ratio and
+/// the legacy layout.
+pub fn ec_shard_config_from(
+ vif: Option<&crate::storage::volume::VifVolumeInfo>,
+ dirs: &[&str],
+ collection: &str,
+ volume_id: VolumeId,
+) -> io::Result<(u32, u32, i64)> {
+ let default_ds = crate::storage::erasure_coding::ec_shard::DATA_SHARDS_COUNT as u32;
+ let default_ps = crate::storage::erasure_coding::ec_shard::PARITY_SHARDS_COUNT as u32;
+ // Sum as u64: counts near the u32 ceiling would wrap and pass the bound.
+ let usable = |ds: u32, ps: u32| {
+ ds > 0 && ps > 0 && (ds as u64 + ps as u64) <= MAX_SHARD_COUNT as u64
+ };
+
+ if let Some(ec) = vif.and_then(|v| v.ec_shard_config.as_ref()) {
+ // A config that is PRESENT but records an impossible ratio is not a
+ // volume to fall back on: dropping through to the sidecar or the
+ // defaults would read uniform shards with the legacy offset math and
+ // answer with the wrong bytes. Only an ENTIRELY absent config means
+ // "this predates the record".
+ if !usable(ec.data_shards, ec.parity_shards) {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ format!(
+ "vif for volume {} records invalid shard counts {}+{}",
+ volume_id.0, ec.data_shards, ec.parity_shards
+ ),
+ ));
+ }
+ // A recorded block size that no encoder could have produced maps
+ // every read to the wrong shard offset. Refuse the mount rather
+ // than serve those bytes or silently pick a layout.
+ validate_block_size(ec.block_size)?;
+ return Ok((ec.data_shards, ec.parity_shards, ec.block_size));
+ }
+ // With no usable vif config the sidecar is the ONLY record of this volume's
+ // layout, so the three answers stay distinct: absent EVERYWHERE means the
+ // volume may genuinely predate the sidecar and legacy is the right guess;
+ // present-but-unusable means the record is corrupt, and answering reads
+ // from a guessed layout returns wrong bytes rather than none.
+ //
+ // Every candidate directory is searched, not just the first: a split
+ // -dir/-dir.idx location keeps the sidecar with the INDEX, and a multi-disk
+ // server may keep it on a sibling. Stopping at the data directory resolved
+ // a 12+4 uniform volume as 10+4 legacy.
+ let mut path = String::new();
+ for candidate in dirs.iter().filter(|d| !d.is_empty()) {
+ let base = crate::storage::volume::volume_file_name(candidate, collection, volume_id);
+ let probe = crate::storage::erasure_coding::ec_bitrot::bitrot_sidecar_path(&base, 0);
+ match std::fs::metadata(&probe) {
+ Err(e) if e.kind() == io::ErrorKind::NotFound => continue,
+ Err(e) => {
+ return Err(io::Error::new(e.kind(), format!("stat {}: {}", probe, e)));
+ }
+ Ok(_) => {
+ path = probe;
+ break;
}
}
}
- (data_shards, parity_shards)
+ if path.is_empty() {
+ return Ok((default_ds, default_ps, 0));
+ }
+ let prot = crate::storage::erasure_coding::ec_bitrot::load_bitrot_sidecar(&path)
+ .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, format!("read {}: {}", path, e)))?;
+ // The un-suffixed sidecar describes generation 0 and nothing else.
+ if prot.generation != 0 {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ format!("{} records generation {}, not generation 0", path, prot.generation),
+ ));
+ }
+ let ec = prot.ec_shard_config.as_ref().ok_or_else(|| {
+ io::Error::new(
+ io::ErrorKind::InvalidData,
+ format!("{} records no EC config", path),
+ )
+ })?;
+ if !usable(ec.data_shards, ec.parity_shards) {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ format!(
+ "{} records invalid shard counts {}+{}",
+ path, ec.data_shards, ec.parity_shards
+ ),
+ ));
+ }
+ validate_block_size(ec.block_size)
+ .map_err(|e| io::Error::new(e.kind(), format!("{}: {}", path, e)))?;
+ Ok((ec.data_shards, ec.parity_shards, ec.block_size))
+}
+
+/// Reports whether a `.vif`-recorded shard block size is one an encoder could
+/// have produced. 0 means the legacy two-tier layout, which is always valid;
+/// anything positive must be a whole number of small blocks, because that is
+/// what `uniform_block_size` rounds to. Mirrors Go's `ValidateBlockSize`.
+pub fn validate_block_size(block_size: i64) -> io::Result<()> {
+ if block_size == 0 {
+ return Ok(());
+ }
+ if block_size < 0
+ || block_size
+ % crate::storage::erasure_coding::ec_shard::ERASURE_CODING_SMALL_BLOCK_SIZE as i64
+ != 0
+ {
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ format!(
+ "invalid shard block size {}: expected 0 (legacy) or a multiple of {}",
+ block_size,
+ crate::storage::erasure_coding::ec_shard::ERASURE_CODING_SMALL_BLOCK_SIZE
+ ),
+ ));
+ }
+ Ok(())
}
impl EcVolume {
@@ -139,7 +357,14 @@ impl EcVolume {
collection: &str,
volume_id: VolumeId,
) -> io::Result<Self> {
- let (data_shards, parity_shards) = read_ec_shard_config(dir, dir_idx, collection, volume_id);
+ // One load of the volume's `.vif`, used for both the shard config and
+ // the version / dat-size fields below.
+ let vif = load_vif_info(dir, dir_idx, collection, volume_id)?;
+ // Both directories: a split -dir/-dir.idx layout keeps the .ecsum with
+ // the INDEX, and it is the only layout record a volume whose vif is
+ // absent or config-free still has.
+ let (data_shards, parity_shards, block_size) =
+ ec_shard_config_from(vif.as_ref(), &[dir, dir_idx], collection, volume_id)?;
let total_shards = (data_shards + parity_shards) as usize;
let mut shards = Vec::with_capacity(total_shards);
@@ -155,12 +380,11 @@ impl EcVolume {
// for shard-size math and by the Store-level prune in
// `store_ec_reconcile.rs` to verify a sibling-disk .dat is
// plausibly the encoding source (#9478).
- let (expire_at_sec, vif_version, vif_dat_file_size, encode_ts_ns) = {
- let vif_path = locate_vif_path(dir, dir_idx, collection, volume_id);
- if let Ok(vif_content) = std::fs::read_to_string(&vif_path) {
- if let Ok(vif_info) =
- serde_json::from_str::<crate::storage::volume::VifVolumeInfo>(&vif_content)
- {
+ let (expire_at_sec, vif_version, vif_dat_file_size, encode_ts_ns) =
+ // Absent everywhere = legacy defaults; a present-but-unreadable or
+ // malformed vif already failed the mount in load_vif_info above.
+ match vif.as_ref() {
+ Some(vif_info) => {
let ver = if vif_info.version > 0 {
Version(vif_info.version as u8)
} else {
@@ -176,13 +400,9 @@ impl EcVolume {
vif_info.dat_file_size,
cfg_encode_ts_ns,
)
- } else {
- (0, Version::current(), 0, 0)
}
- } else {
- (0, Version::current(), 0, 0)
- }
- };
+ None => (0, Version::current(), 0, 0),
+ };
let mut vol = EcVolume {
volume_id,
@@ -194,6 +414,7 @@ impl EcVolume {
dat_file_size: vif_dat_file_size,
data_shards,
parity_shards,
+ block_size,
ecx_file: None,
ecx_file_size: 0,
ecj_file: None,
@@ -254,17 +475,33 @@ impl EcVolume {
// Seed the in-memory deleted set from the journal.
vol.load_deleted_needles_from_ecj()?;
- // Load the generation-0 EC bitrot checksum sidecar (optional; best-effort).
- vol.load_active_bitrot_sidecar();
+ // Load the generation-0 EC bitrot checksum sidecar. Optional, except
+ // when it contradicts the volume's own geometry — see
+ // load_bitrot_for_generation.
+ vol.load_active_bitrot_sidecar(&[])?;
Ok(vol)
}
+ /// Re-resolve the checksum sidecar for a volume that is already mounted. A
+ /// shard delivery can bring the manifest with it, and the receive path only
+ /// writes the file, so without this the in-memory volume keeps the
+ /// protection state it resolved at mount (off) until a remount.
+ pub fn reload_bitrot_sidecar(&mut self, additional_dirs: &[String]) {
+ if let Err(e) = self.load_active_bitrot_sidecar(additional_dirs) {
+ tracing::warn!(
+ volume_id = self.volume_id.0,
+ error = %e,
+ "reload bitrot sidecar",
+ );
+ }
+ }
+
/// Load the generation-0 checksum sidecar into `self.bitrot`/`self.bitrot_status`.
/// OSS only produces generation-0 (fresh-encode) sidecars, mirroring Go's
/// `loadActiveBitrotSidecar`.
- fn load_active_bitrot_sidecar(&mut self) {
- self.load_bitrot_for_generation(0);
+ fn load_active_bitrot_sidecar(&mut self, additional_dirs: &[String]) -> io::Result<()> {
+ self.load_bitrot_for_generation(0, additional_dirs)
}
/// Load and validate the sidecar describing `generation`, setting
@@ -272,11 +509,60 @@ impl EcVolume {
/// => `Off` (protection off, not corruption); self-integrity or manifest
/// failure => `Invalid` with a warning (protection off pending repair); usable
/// => `On`. Mirrors Go's `loadBitrotForGeneration`.
- fn load_bitrot_for_generation(&mut self, generation: u32) {
+ fn load_bitrot_for_generation(&mut self, generation: u32, additional_dirs: &[String]) -> io::Result<()> {
use crate::storage::erasure_coding::ec_bitrot;
- let base = self.base_name();
- let path = ec_bitrot::bitrot_sidecar_path(&base, generation);
+ // Data base then index base, matching Go's findBitrotSidecar. A split
+ // -dir/-dir.idx location keeps the sidecar with the INDEX, and on a
+ // multi-disk server sharing one index directory that is the only copy
+ // the per-disk runtimes other than the first can see — searching the
+ // data base alone left them reporting no protection at all.
+ let data_path = ec_bitrot::bitrot_sidecar_path(&self.base_name(), generation);
+ let idx_path = ec_bitrot::bitrot_sidecar_path(&self.idx_base_name(), generation);
+ // Sibling disks last: startup mirroring gives each shard-bearing disk
+ // its own .ecx/.ecj/.vif but not the sidecar, and a delivery lands
+ // exactly one copy, so a runtime restricted to its own two directories
+ // would report no protection however often it reloaded.
+ let path = std::iter::once(data_path.clone())
+ .chain(std::iter::once(idx_path))
+ .chain(additional_dirs.iter().filter(|d| !d.is_empty()).map(|d| {
+ ec_bitrot::bitrot_sidecar_path(
+ &crate::storage::volume::volume_file_name(d, &self.collection, self.volume_id),
+ generation,
+ )
+ }))
+ .find(|p| std::path::Path::new(p).exists())
+ .unwrap_or(data_path);
let loaded = ec_bitrot::load_bitrot_sidecar(&path);
+ // A sidecar written for THIS generation that contradicts the volume's
+ // geometry is not "no protection" — it says the layout the volume is
+ // about to serve reads with is wrong. Fail the mount.
+ if let Ok(prot) = &loaded {
+ if prot.generation == generation
+ && !ec_bitrot::geometry_matches(
+ prot,
+ self.data_shards as usize,
+ self.parity_shards as usize,
+ self.block_size,
+ )
+ {
+ let cfg = prot.ec_shard_config.as_ref();
+ return Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ format!(
+ "ec volume {} generation {}: {} records layout {}+{} block {} but the volume is mounted as {}+{} block {}; refusing to serve one of the two layouts",
+ self.volume_id.0,
+ generation,
+ path,
+ cfg.map(|c| c.data_shards).unwrap_or(0),
+ cfg.map(|c| c.parity_shards).unwrap_or(0),
+ cfg.map(|c| c.block_size).unwrap_or(0),
+ self.data_shards,
+ self.parity_shards,
+ self.block_size,
+ ),
+ ));
+ }
+ }
let status = ec_bitrot::resolve_status(
&loaded,
generation,
@@ -297,6 +583,7 @@ impl EcVolume {
);
}
}
+ Ok(())
}
/// The active-generation bitrot protection AND its status (cached at mount),
@@ -765,6 +1052,26 @@ impl EcVolume {
Ok(None)
}
+ /// Large-block length of this volume's shard layout (the uniform block
+ /// size when set, the legacy 1GiB otherwise). Mirrors Go's
+ /// ECContext.LargeBlockSize.
+ pub fn large_block_size(&self) -> i64 {
+ if self.block_size > 0 {
+ self.block_size
+ } else {
+ ERASURE_CODING_LARGE_BLOCK_SIZE as i64
+ }
+ }
+
+ /// Small-block length of this volume's shard layout.
+ pub fn small_block_size(&self) -> i64 {
+ if self.block_size > 0 {
+ self.block_size
+ } else {
+ ERASURE_CODING_SMALL_BLOCK_SIZE as i64
+ }
+ }
+
/// Locate the EC shard intervals needed to read a needle.
/// Locate the EC shard intervals covering a needle at `actual_offset` whose
/// index size is `size`. Mirrors Go's EcVolume.LocateEcShardNeedleInterval.
@@ -783,7 +1090,27 @@ impl EcVolume {
};
// locate_data wants the on-disk size (header+body+checksum+timestamp+padding).
let actual = get_actual_size(size, self.version);
- ec_locate::locate_data(actual_offset, Size(actual as i32), shard_size, self.data_shards)
+ ec_locate::locate_data(
+ actual_offset,
+ Size(actual as i32),
+ shard_size,
+ self.data_shards,
+ self.large_block_size(),
+ self.small_block_size(),
+ )
+ }
+
+ /// Resolve an interval against this volume's shard block layout. Mirrors
+ /// Go's EcVolume.IntervalToShardIdAndOffset.
+ pub fn interval_to_shard_id_and_offset(
+ &self,
+ interval: &ec_locate::Interval,
+ ) -> (ShardId, i64) {
+ interval.to_shard_id_and_offset(
+ self.data_shards,
+ self.large_block_size(),
+ self.small_block_size(),
+ )
}
pub fn locate_needle(
@@ -829,7 +1156,7 @@ impl EcVolume {
let mut bytes = Vec::with_capacity(actual_size);
for interval in &intervals {
- let (shard_id, shard_offset) = interval.to_shard_id_and_offset(self.data_shards);
+ let (shard_id, shard_offset) = self.interval_to_shard_id_and_offset(interval);
let shard = self
.shards
.get(shard_id as usize)
@@ -989,7 +1316,7 @@ impl EcVolume {
// A needle is verifiable locally only if every shard it spans is local;
// when any is remote, skip the reassembly buffer entirely.
let has_remote_chunks = locations.iter().any(|iv| {
- let (sid, _) = iv.to_shard_id_and_offset(self.data_shards);
+ let (sid, _) = self.interval_to_shard_id_and_offset(iv);
self.shards.get(sid as usize).and_then(|s| s.as_ref()).is_none()
});
let mut read: i64 = 0;
@@ -1001,7 +1328,7 @@ impl EcVolume {
let mut local_shard_ids: Vec<ShardId> = Vec::new();
for (i, iv) in locations.iter().enumerate() {
- let (sid, soffset) = iv.to_shard_id_and_offset(self.data_shards);
+ let (sid, soffset) = self.interval_to_shard_id_and_offset(iv);
let ssize = iv.size;
let shard = match self.shards.get(sid as usize).and_then(|s| s.as_ref()) {
Some(s) => s,
@@ -1800,8 +2127,14 @@ mod tests {
assert_eq!(vol.parity_shards, 3);
}
+ /// A vif that RECORDS a ratio no volume could have must fail the mount.
+ /// Substituting the default 10+4 (and with it the legacy block layout)
+ /// would read a uniform volume's shards at the wrong offsets and answer
+ /// with the wrong bytes; only an entirely absent config means "this
+ /// predates the record", which
+ /// `test_ec_volume_absent_vif_config_uses_defaults` covers.
#[test]
- fn test_ec_volume_invalid_vif_config_falls_back_to_defaults() {
+ fn test_ec_volume_invalid_vif_config_fails_the_mount() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
write_ecx_file(dir, "pics", VolumeId(1), &[]);
@@ -1822,9 +2155,27 @@ mod tests {
)
.unwrap();
+ // EcVolume has no Debug impl, so match rather than expect_err.
+ match EcVolume::new(dir, dir, "pics", VolumeId(1)) {
+ Ok(_) => panic!("an impossible ratio must fail the mount"),
+ Err(e) => assert_eq!(e.kind(), io::ErrorKind::InvalidData, "got {e}"),
+ }
+ }
+
+ /// The compatibility case the check above must not swallow: a vif with no
+ /// EC config at all predates the record, and mounts on the defaults.
+ #[test]
+ fn test_ec_volume_absent_vif_config_uses_defaults() {
+ let tmp = TempDir::new().unwrap();
+ let dir = tmp.path().to_str().unwrap();
+ write_ecx_file(dir, "pics", VolumeId(1), &[]);
+ let base = crate::storage::volume::volume_file_name(dir, "pics", VolumeId(1));
+ std::fs::write(format!("{}.vif", base), r#"{"version":3}"#).unwrap();
+
let vol = EcVolume::new(dir, dir, "pics", VolumeId(1)).unwrap();
assert_eq!(vol.data_shards, DATA_SHARDS_COUNT as u32);
assert_eq!(vol.parity_shards, PARITY_SHARDS_COUNT as u32);
+ assert_eq!(vol.block_size, 0, "legacy layout for a pre-record volume");
}
#[test]
@@ -1973,3 +2324,370 @@ mod tests {
);
}
}
+
+#[cfg(test)]
+mod uniform_layout_tests {
+ use super::*;
+ use crate::storage::needle_map::NeedleMapKind;
+ use crate::storage::volume::{VifEcShardConfig, VifVolumeInfo, Volume};
+ use tempfile::TempDir;
+
+ // Write ~26MB of needles so the uniform block size (3MB) diverges from the
+ // legacy layout, encode, and verify EcVolume reads every needle back
+ // through the .vif-recorded geometry. A legacy-encoded fixture (no .vif
+ // block size) must keep reading through the legacy interpretation.
+ #[test]
+ fn test_read_needles_uniform_and_legacy_layouts() {
+ for legacy in [false, true] {
+ let tmp = TempDir::new().unwrap();
+ let dir = tmp.path().to_str().unwrap();
+ let vid = VolumeId(8);
+
+ let mut v = Volume::new(
+ dir,
+ dir,
+ "",
+ vid,
+ NeedleMapKind::InMemory,
+ None,
+ None,
+ 0,
+ Version::current(),
+ )
+ .unwrap();
+ let mut expected: Vec<(NeedleId, Vec<u8>)> = Vec::new();
+ for i in 1u64..=11 {
+ let size = if i <= 5 { 4 << 20 } else { 1 << 20 };
+ let data: Vec<u8> = (0..size)
+ .map(|b| ((b as u64).wrapping_mul(2654435761).wrapping_add(i) >> 8) as u8)
+ .collect();
+ let mut n = Needle {
+ id: NeedleId(i),
+ cookie: Cookie(i as u32),
+ data: data.clone(),
+ data_size: data.len() as u32,
+ ..Needle::default()
+ };
+ v.write_needle(&mut n, true, false).unwrap();
+ expected.push((NeedleId(i), data));
+ }
+ v.sync_to_disk().unwrap();
+ let dat_size = v.dat_file_size().unwrap() as i64;
+ v.close();
+
+ let block_size = if legacy {
+ // Legacy fixture: two-tier encode plus a .vif without a block
+ // size, the state every pre-upgrade EC volume is in.
+ use crate::storage::erasure_coding::ec_bitrot::{
+ ShardChecksumBuilder, DEFAULT_BITROT_BLOCK_SIZE,
+ };
+ use reed_solomon_erasure::galois_8::ReedSolomon;
+ let base = crate::storage::volume::volume_file_name(dir, "", vid);
+ crate::storage::erasure_coding::ec_encoder::write_sorted_ecx_from_idx(
+ &format!("{}.idx", base),
+ &format!("{}.ecx", base),
+ )
+ .unwrap();
+ let dat_file = std::fs::File::open(format!("{}.dat", base)).unwrap();
+ let rs = ReedSolomon::new(10, 4).unwrap();
+ let mut shards: Vec<EcVolumeShard> = (0..14u8)
+ .map(|i| EcVolumeShard::new(dir, "", vid, i))
+ .collect();
+ for shard in &mut shards {
+ shard.create().unwrap();
+ }
+ let mut builders: Vec<ShardChecksumBuilder> = (0..14)
+ .map(|_| ShardChecksumBuilder::new(DEFAULT_BITROT_BLOCK_SIZE as i64))
+ .collect();
+ crate::storage::erasure_coding::ec_encoder::encode_dat_file(
+ &dat_file,
+ dat_size,
+ &rs,
+ &mut shards,
+ &mut builders,
+ 10,
+ 4,
+ 256 * 1024,
+ ERASURE_CODING_LARGE_BLOCK_SIZE,
+ ERASURE_CODING_SMALL_BLOCK_SIZE,
+ )
+ .unwrap();
+ for shard in &mut shards {
+ shard.close();
+ }
+ 0
+ } else {
+ let bs = crate::storage::erasure_coding::ec_encoder::write_ec_files(
+ dir, dir, "", vid, 10, 4,
+ )
+ .unwrap();
+ assert!(
+ bs > ERASURE_CODING_SMALL_BLOCK_SIZE as i64,
+ "block size {} does not diverge from the legacy layout",
+ bs
+ );
+ bs
+ };
+
+ let base = crate::storage::volume::volume_file_name(dir, "", vid);
+ let vif = VifVolumeInfo {
+ version: Version::current().0 as u32,
+ dat_file_size: dat_size,
+ ec_shard_config: Some(VifEcShardConfig {
+ data_shards: 10,
+ parity_shards: 4,
+ block_size,
+ ..Default::default()
+ }),
+ ..Default::default()
+ };
+ std::fs::write(
+ format!("{}.vif", base),
+ serde_json::to_string_pretty(&vif).unwrap(),
+ )
+ .unwrap();
+ std::fs::remove_file(format!("{}.dat", base)).unwrap();
+ std::fs::remove_file(format!("{}.idx", base)).unwrap();
+
+ let mut vol = EcVolume::new(dir, dir, "", vid).unwrap();
+ assert_eq!(vol.block_size, block_size, "block size not loaded from .vif");
+ for i in 0..10u8 {
+ vol.add_shard(EcVolumeShard::new(dir, "", vid, i)).unwrap();
+ }
+
+ for (id, data) in &expected {
+ let n = vol
+ .read_ec_shard_needle(*id)
+ .unwrap()
+ .unwrap_or_else(|| panic!("needle {} not found (legacy={})", id.0, legacy));
+ assert_eq!(
+ n.data, *data,
+ "needle {} data mismatch (legacy={})",
+ id.0, legacy
+ );
+ }
+ }
+ }
+
+ // A present-but-malformed .vif must FAIL the mount: every new encode
+ // records a positive uniform block size there, and silently defaulting
+ // to the legacy layout would serve those shards with the wrong offset
+ // math. Absence stays legal — legacy volumes predate the sidecar.
+ #[test]
+ fn new_fails_on_malformed_vif() {
+ let tmp = tempfile::TempDir::new().unwrap();
+ let dir = tmp.path().to_str().unwrap();
+ let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
+ std::fs::write(format!("{}.ecx", base), b"").unwrap();
+ std::fs::write(format!("{}.vif", base), b"not json").unwrap();
+ let res = EcVolume::new(dir, dir, "", VolumeId(1));
+ assert!(res.is_err(), "mount over a malformed .vif must fail");
+ }
+
+ #[test]
+ fn new_absent_vif_mounts_with_defaults() {
+ let tmp = tempfile::TempDir::new().unwrap();
+ let dir = tmp.path().to_str().unwrap();
+ let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(1));
+ std::fs::write(format!("{}.ecx", base), b"").unwrap();
+ let ev = EcVolume::new(dir, dir, "", VolumeId(1)).unwrap();
+ assert_eq!(ev.block_size, 0, "legacy mount must use the legacy layout");
+ }
+
+ /// A rebuild lands on one disk, but the volume's metadata may live on
+ /// another. Reading only the selected directory returns the default 10+4
+ /// and the legacy block layout, which reconstructs a custom-ratio or
+ /// uniform volume through the wrong matrix.
+ fn seed_uniform_sidecar(base: &str, ds: u32, ps: u32, block: i64) {
+ use crate::pb::volume_server_pb::{ChecksumAlgorithm, EcBitrotProtection, EcShardChecksums};
+ use crate::storage::erasure_coding::ec_bitrot;
+ let prot = EcBitrotProtection {
+ algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
+ block_size: ec_bitrot::DEFAULT_BITROT_BLOCK_SIZE as u32,
+ generation: 0,
+ ec_shard_config: Some(ec_bitrot::ec_shard_config(ds, ps, block)),
+ shards: (0..(ds + ps))
+ .map(|shard_id| EcShardChecksums {
+ shard_id,
+ covered_size: 4,
+ block_crc32c: vec![0u8; 4],
+ })
+ .collect(),
+ encode_uuid: vec![0u8; 16],
+ };
+ ec_bitrot::save_bitrot_sidecar(&ec_bitrot::bitrot_sidecar_path(base, 0), &prot).unwrap();
+ }
+
+ fn seed_config_free_vif(base: &str) {
+ std::fs::write(
+ format!("{}.vif", base),
+ r#"{"version":3,"datFileSize":0}"#,
+ )
+ .unwrap();
+ }
+
+ // A .vif that omits ecShardConfig answers nothing about the layout, so it
+ // must not short-circuit the sidecar search. Returning as soon as any
+ // parseable vif turned up resolved a 12+4 uniform volume as 10+4 legacy.
+ #[test]
+ fn read_ec_shard_config_falls_back_when_the_vif_has_no_config() {
+ let d = tempfile::TempDir::new().unwrap();
+ let dir = d.path().to_str().unwrap();
+ let base = crate::storage::volume::volume_file_name(dir, "", VolumeId(11));
+ seed_config_free_vif(&base);
+ seed_uniform_sidecar(&base, 12, 4, 3 * 1024 * 1024);
+
+ let got = read_ec_shard_config(dir, dir, "", VolumeId(11)).unwrap();
+ assert_eq!(got, (12, 4, 3 * 1024 * 1024));
+ }
+
+ // Split -dir/-dir.idx: the vif and the sidecar both live with the INDEX,
+ // and the resolver was told to report the DATA directory, so the fallback
+ // looked in a directory holding neither.
+ #[test]
+ fn read_ec_shard_config_finds_a_config_free_vifs_sidecar_in_the_index_dir() {
+ let data = tempfile::TempDir::new().unwrap();
+ let idx = tempfile::TempDir::new().unwrap();
+ let (dir, dir_idx) = (data.path().to_str().unwrap(), idx.path().to_str().unwrap());
+ let idx_base = crate::storage::volume::volume_file_name(dir_idx, "", VolumeId(12));
+ seed_config_free_vif(&idx_base);
+ seed_uniform_sidecar(&idx_base, 12, 4, 3 * 1024 * 1024);
+
+ let got = read_ec_shard_config(dir, dir_idx, "", VolumeId(12)).unwrap();
+ assert_eq!(got, (12, 4, 3 * 1024 * 1024));
+ }
+
+ // Same gap in the cross-disk resolver the rebuild uses.
+ #[test]
+ fn read_ec_shard_config_across_dirs_falls_back_when_the_vif_has_no_config() {
+ let data = tempfile::TempDir::new().unwrap();
+ let sib = tempfile::TempDir::new().unwrap();
+ let (dir, sibling) = (data.path().to_str().unwrap(), sib.path().to_str().unwrap());
+ seed_config_free_vif(&crate::storage::volume::volume_file_name(dir, "", VolumeId(13)));
+ seed_uniform_sidecar(
+ &crate::storage::volume::volume_file_name(sibling, "", VolumeId(13)),
+ 12,
+ 4,
+ 3 * 1024 * 1024,
+ );
+
+ let got = read_ec_shard_config_across_dirs(
+ dir,
+ dir,
+ &[sibling.to_string()],
+ "",
+ VolumeId(13),
+ )
+ .unwrap();
+ assert_eq!(got, (12, 4, 3 * 1024 * 1024));
+ }
+
+ // Absence stays legal: a volume predating both records is genuinely legacy.
+ #[test]
+ fn read_ec_shard_config_config_free_vif_without_a_sidecar_is_legacy() {
+ let d = tempfile::TempDir::new().unwrap();
+ let dir = d.path().to_str().unwrap();
+ seed_config_free_vif(&crate::storage::volume::volume_file_name(dir, "", VolumeId(14)));
+ let got = read_ec_shard_config(dir, dir, "", VolumeId(14)).unwrap();
+ assert_eq!(got, (10, 4, 0));
+ }
+
+ #[test]
+ fn read_ec_shard_config_finds_a_sibling_disks_vif() {
+ let a = tempfile::TempDir::new().unwrap();
+ let b = tempfile::TempDir::new().unwrap();
+ let (rebuild, sibling) = (a.path().to_str().unwrap(), b.path().to_str().unwrap());
+ let base = crate::storage::volume::volume_file_name(sibling, "", VolumeId(3));
+ std::fs::write(
+ format!("{}.vif", base),
+ r#"{"version":3,"ecShardConfig":{"dataShards":12,"parityShards":4,"blockSize":3145728}}"#,
+ )
+ .unwrap();
+
+ let (ds, ps, bs) = read_ec_shard_config_across_dirs(
+ rebuild,
+ rebuild,
+ &[sibling.to_string()],
+ "",
+ VolumeId(3),
+ )
+ .unwrap();
+ assert_eq!((ds, ps, bs), (12, 4, 3 * 1024 * 1024));
+ }
+
+ /// Same, for the sidecar: with no .vif anywhere it is the surviving record
+ /// of the geometry, wherever it sits.
+ // A split -dir/-dir.idx location keeps its metadata with the INDEX, and
+ // callers leave their own index directory out of other_dirs because it is
+ // passed separately. With no .vif anywhere the generation-0 .ecsum is the
+ // only record of the layout, so missing that directory resolves a 12+4
+ // uniform volume to 10+4 legacy and reconstructs through the wrong matrix.
+ #[test]
+ fn read_ec_shard_config_finds_the_sidecar_in_its_own_index_dir() {
+ use crate::pb::volume_server_pb::{ChecksumAlgorithm, EcBitrotProtection, EcShardChecksums};
+ use crate::storage::erasure_coding::ec_bitrot;
+
+ let data = tempfile::TempDir::new().unwrap();
+ let idx = tempfile::TempDir::new().unwrap();
+ let (dir, dir_idx) = (
+ data.path().to_str().unwrap(),
+ idx.path().to_str().unwrap(),
+ );
+ let base = crate::storage::volume::volume_file_name(dir_idx, "", VolumeId(7));
+ let prot = EcBitrotProtection {
+ algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
+ block_size: ec_bitrot::DEFAULT_BITROT_BLOCK_SIZE as u32,
+ generation: 0,
+ ec_shard_config: Some(ec_bitrot::ec_shard_config(12, 4, 3 * 1024 * 1024)),
+ shards: (0..16u32)
+ .map(|shard_id| EcShardChecksums {
+ shard_id,
+ covered_size: 4,
+ block_crc32c: vec![0u8; 4],
+ })
+ .collect(),
+ encode_uuid: vec![0u8; 16],
+ };
+ ec_bitrot::save_bitrot_sidecar(&ec_bitrot::bitrot_sidecar_path(&base, 0), &prot).unwrap();
+
+ let (ds, ps, bs) =
+ read_ec_shard_config_across_dirs(dir, dir_idx, &[], "", VolumeId(7)).unwrap();
+ assert_eq!((ds, ps, bs), (12, 4, 3 * 1024 * 1024));
+ }
+
+ #[test]
+ fn read_ec_shard_config_finds_a_sibling_disks_sidecar() {
+ use crate::pb::volume_server_pb::{ChecksumAlgorithm, EcBitrotProtection, EcShardChecksums};
+ use crate::storage::erasure_coding::ec_bitrot;
+
+ let a = tempfile::TempDir::new().unwrap();
+ let b = tempfile::TempDir::new().unwrap();
+ let (rebuild, sibling) = (a.path().to_str().unwrap(), b.path().to_str().unwrap());
+ let base = crate::storage::volume::volume_file_name(sibling, "", VolumeId(4));
+ let prot = EcBitrotProtection {
+ algorithm: ChecksumAlgorithm::ChecksumCrc32c as i32,
+ block_size: ec_bitrot::DEFAULT_BITROT_BLOCK_SIZE as u32,
+ generation: 0,
+ ec_shard_config: Some(ec_bitrot::ec_shard_config(12, 4, 3 * 1024 * 1024)),
+ shards: (0..16u32)
+ .map(|shard_id| EcShardChecksums {
+ shard_id,
+ covered_size: 4,
+ block_crc32c: vec![0u8; 4],
+ })
+ .collect(),
+ encode_uuid: vec![0u8; 16],
+ };
+ ec_bitrot::save_bitrot_sidecar(&ec_bitrot::bitrot_sidecar_path(&base, 0), &prot).unwrap();
+
+ let (ds, ps, bs) = read_ec_shard_config_across_dirs(
+ rebuild,
+ rebuild,
+ &[sibling.to_string()],
+ "",
+ VolumeId(4),
+ )
+ .unwrap();
+ assert_eq!((ds, ps, bs), (12, 4, 3 * 1024 * 1024));
+ }
+}
diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs
index 644c6b76df..b458efb472 100644
--- a/seaweed-volume/src/storage/store.rs
+++ b/seaweed-volume/src/storage/store.rs
@@ -804,6 +804,37 @@ impl Store {
}
/// Find an EC volume across all locations (mutable).
+ /// Every per-disk `EcVolume` this store maps for `vid`. A vid can mount on
+ /// N disks as N distinct runtimes, and the first-match `find_ec_volume_mut`
+ /// hides the siblings — so anything that has to reach the whole volume,
+ /// rather than any one runtime of it, iterates this instead.
+ /// Every directory on this server that could hold an EC volume's metadata —
+ /// each disk's data and index directory. Startup mirroring gives each
+ /// shard-bearing disk its own .ecx/.ecj/.vif, but the checksum sidecar is
+ /// not mirrored and a repair delivers exactly one copy, so a runtime looking
+ /// only at its own two directories cannot see it. Handing this list to the
+ /// sidecar resolution keeps one authoritative copy reachable from every
+ /// runtime rather than duplicating a file that is rewritten as shards are
+ /// repaired and generations published.
+ pub fn ec_metadata_dirs(&self) -> Vec<String> {
+ let mut dirs: Vec<String> = Vec::with_capacity(self.locations.len() * 2);
+ for loc in &self.locations {
+ for dir in [&loc.directory, &loc.idx_directory] {
+ if !dir.is_empty() && !dirs.iter().any(|d| d == dir) {
+ dirs.push(dir.clone());
+ }
+ }
+ }
+ dirs
+ }
+
+ pub fn find_all_ec_volumes_mut(&mut self, vid: VolumeId) -> Vec<&mut EcVolume> {
+ self.locations
+ .iter_mut()
+ .filter_map(|loc| loc.find_ec_volume_mut(vid))
+ .collect()
+ }
+
pub fn find_ec_volume_mut(&mut self, vid: VolumeId) -> Option<&mut EcVolume> {
for loc in &mut self.locations {
if let Some(ecv) = loc.find_ec_volume_mut(vid) {
diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs
index 64572c3458..6373c9eda8 100644
--- a/seaweed-volume/src/storage/volume.rs
+++ b/seaweed-volume/src/storage/volume.rs
@@ -194,6 +194,10 @@ pub struct VifEcShardConfig {
/// so a read served from a different run's shard is rejected.
#[serde(default, rename = "encodeTsNs", with = "string_or_i64")]
pub encode_ts_ns: i64,
+ /// Uniform block layout: each shard is a single contiguous block of this
+ /// many bytes. 0 = legacy 1GiB/1MiB two-tier layout.
+ #[serde(default, rename = "blockSize", with = "string_or_i64")]
+ pub block_size: i64,
}
/// Serde-compatible representation of OldVersionVolumeInfo for legacy .vif JSON deserialization.
@@ -288,6 +292,7 @@ impl VifVolumeInfo {
data_shards: c.data_shards,
parity_shards: c.parity_shards,
encode_ts_ns: c.encode_ts_ns,
+ block_size: c.block_size,
}),
read_only_can_delete: pb.read_only_can_delete,
}
@@ -320,6 +325,7 @@ impl VifVolumeInfo {
data_shards: c.data_shards,
parity_shards: c.parity_shards,
encode_ts_ns: c.encode_ts_ns,
+ block_size: c.block_size,
}
}),
read_only_can_delete: self.read_only_can_delete,
diff --git a/test/plugin_workers/fake_volume_server.go b/test/plugin_workers/fake_volume_server.go
index 103515ce6c..1ea58276e2 100644
--- a/test/plugin_workers/fake_volume_server.go
+++ b/test/plugin_workers/fake_volume_server.go
@@ -15,6 +15,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)
@@ -339,7 +340,14 @@ func (v *VolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_serve
}
}
- resp := &volume_server_pb.VolumeEcShardsInfoResponse{}
+ // Answer with the layout out of the .vif that was actually delivered here,
+ // the way a real holder answers from the context it mounted the shards
+ // with. A coordinator uses this to tell a server that understands the
+ // shard block layout from one that never knew the field, so a fake that
+ // always reported "unset" would look like a pre-upgrade server.
+ resp := &volume_server_pb.VolumeEcShardsInfoResponse{
+ EcShardConfig: v.ecShardConfigFromVif(req.VolumeId),
+ }
prefix := fmt.Sprintf("%d.ec", req.VolumeId)
entries, _ := os.ReadDir(v.baseDir)
for _, entry := range entries {
@@ -372,6 +380,16 @@ func (v *VolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_serve
return resp, nil
}
+// ecShardConfigFromVif reads the EC layout out of the .vif this server was
+// given, which distribution ships to every holder alongside its shards.
+func (v *VolumeServer) ecShardConfigFromVif(volumeID uint32) *volume_server_pb.EcShardConfig {
+ vi, _, found, err := volume_info.MaybeLoadVolumeInfo(v.filePath(volumeID, ".vif"))
+ if err != nil || !found {
+ return nil
+ }
+ return vi.GetEcShardConfig()
+}
+
func (v *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
v.mu.Lock()
v.deleteRequests = append(v.deleteRequests, req)
diff --git a/weed/command/fix.go b/weed/command/fix.go
index 6c24dd3204..301bfd0903 100644
--- a/weed/command/fix.go
+++ b/weed/command/fix.go
@@ -327,9 +327,15 @@ func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int6
// Priority: explicit flags > existing .vif > defaults (10+4).
vifName := base + ".vif"
vifExists := util.FileExists(vifName)
+ // Whether the on-disk .vif actually answers the layout question. An empty
+ // stub (MaybeLoadVolumeInfo reads it as absent) or one with no EC config
+ // exists but tells us nothing, and the recovered layout must still be
+ // written back over it.
+ vifUsable := false
dataShards := erasure_coding.DataShardsCount
parityShards := erasure_coding.ParityShardsCount
var datFileSize int64
+ blockSize := int64(-1) // the shard block layout; 0 legacy, >0 uniform, <0 unknown
if vifExists {
// MaybeLoadVolumeInfo returns a non-nil error when the .vif exists but
// cannot be read or unmarshalled; fail loudly rather than silently
@@ -338,13 +344,66 @@ func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int6
fail(fmt.Errorf("volume %d: read %s: %w", volumeId, vifName, loadErr))
return
} else if found && vi != nil {
- if cfg := vi.GetEcShardConfig(); cfg != nil && cfg.GetDataShards() > 0 {
+ // A partial config (parity 0, or counts past the shard ceiling)
+ // describes no layout: leave the sentinel unknown and let the
+ // sidecar / dual scan below answer, and rewrite the .vif at the end
+ // rather than trusting it.
+ cfg := vi.GetEcShardConfig()
+ if cfg != nil && erasure_coding.ValidEcShardCounts(cfg.GetDataShards(), cfg.GetParityShards()) {
dataShards = int(cfg.GetDataShards())
parityShards = int(cfg.GetParityShards())
+ // Only an EC config answers the layout question. Reading 0 off a
+ // vif with no config would assert "legacy" and suppress both the
+ // .ecsum fallback and the dual-layout scan below; leave the
+ // sentinel at -1 (unknown) instead.
+ // A recorded block size no encoder could have produced is not
+ // an answer: a positive one would pin the scan to a geometry
+ // that de-stripes to garbage, and a negative one would leave
+ // the invalid .vif in place after the dual scan recovers the
+ // real layout. Leave the sentinel unknown and rewrite the file
+ // at the end.
+ if bs := cfg.GetBlockSize(); erasure_coding.ValidateBlockSize(bs) == nil {
+ blockSize = bs
+ vifUsable = true
+ } else {
+ glog.Warningf("volume %d: %s records an invalid shard block size %d; recovering the layout by scan and rewriting it",
+ volumeId, vifName, bs)
+ }
}
datFileSize = vi.GetDatFileSize()
}
}
+ // The bitrot sidecar records the same EC config at encode time; use it when
+ // the .vif is gone. A uniform-layout volume cannot be de-striped correctly
+ // without its block size.
+ if blockSize < 0 {
+ sidecarPath := erasure_coding.BitrotSidecarPath(base, 0)
+ if prot, serr := erasure_coding.LoadBitrotSidecar(sidecarPath); serr == nil {
+ // The sidecar is about to pin the reconstruction to one geometry
+ // instead of letting the dual scan decide, so every field it
+ // contributes has to hold up: it must describe generation 0 (the
+ // shards being read), a complete in-range ratio, and a block size
+ // an encoder could have produced. Anything less leaves the layout
+ // unknown, which is the answer that still recovers by scanning.
+ cfg := prot.GetEcShardConfig()
+ ds, ps := int(cfg.GetDataShards()), int(cfg.GetParityShards())
+ switch {
+ case prot.GetGeneration() != 0:
+ glog.Warningf("volume %d: %s records generation %d, not the generation-0 shards; ignoring it",
+ volumeId, sidecarPath, prot.GetGeneration())
+ case !erasure_coding.ValidEcShardCounts(cfg.GetDataShards(), cfg.GetParityShards()):
+ glog.Warningf("volume %d: %s records invalid shard counts %d+%d; ignoring it",
+ volumeId, sidecarPath, ds, ps)
+ case erasure_coding.ValidateBlockSize(cfg.GetBlockSize()) != nil:
+ glog.Warningf("volume %d: %s records an invalid shard block size %d; ignoring it",
+ volumeId, sidecarPath, cfg.GetBlockSize())
+ default:
+ dataShards = ds
+ parityShards = ps
+ blockSize = cfg.GetBlockSize()
+ }
+ }
+ }
if *fixEcDataShards > 0 {
dataShards = *fixEcDataShards
}
@@ -381,6 +440,9 @@ func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int6
}
if !dataComplete {
ctx := &erasure_coding.ECContext{DataShards: dataShards, ParityShards: parityShards}
+ if blockSize > 0 {
+ ctx.BlockSize = blockSize
+ }
glog.Infof("volume %d: %d/%d shards present; reconstructing missing shards (%s) before index rebuild", volumeId, presentCount, dataShards+parityShards, ctx.String())
if _, err := erasure_coding.RebuildEcFiles(base, ctx, *fixEcUnsafeIgnoreSidecar); err != nil {
fail(fmt.Errorf("volume %d: reconstruct missing shards from %d survivors: %w", volumeId, presentCount, err))
@@ -407,37 +469,114 @@ func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int6
glog.V(0).Infof("volume %d: no .dat size in .vif; reconstructing padded .dat (%d bytes) from %d data shards", volumeId, reconstructSize, dataShards)
}
- // De-stripe the data shards into a temporary .dat next to the shards.
+ // De-stripe the data shards into a temporary .dat next to the shards and
+ // scan it into a fresh .ecx. With the layout unknown, de-stripe under both
+ // candidate layouts and keep the one whose needle chain scans furthest.
+ type layoutCandidate struct {
+ name string
+ large, small int64
+ }
+ var candidates []layoutCandidate
+ switch {
+ case blockSize > 0:
+ candidates = []layoutCandidate{{"uniform", blockSize, blockSize}}
+ case blockSize == 0:
+ candidates = []layoutCandidate{{"legacy", erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize}}
+ default:
+ candidates = []layoutCandidate{
+ {"legacy", erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize},
+ }
+ // A uniform-layout shard is exactly one block long, and every block
+ // size an encoder can produce is a whole number of small blocks. An
+ // extent that is not — a truncated or partially copied shard — could
+ // not have come from a uniform encode, and recording it would write a
+ // .vif that ValidateBlockSize refuses on the next mount: the volume
+ // this tool was run to rescue would never open again.
+ if erasure_coding.ValidateBlockSize(shardSize) == nil && shardSize > 0 {
+ candidates = append(candidates, layoutCandidate{"uniform", shardSize, shardSize})
+ glog.Infof("volume %d: no .vif or .ecsum records the shard block layout; trying both", volumeId)
+ } else {
+ glog.Infof("volume %d: no .vif or .ecsum records the shard block layout, and the %d-byte shard extent is not a whole number of %d-byte blocks; trying the legacy layout only",
+ volumeId, shardSize, erasure_coding.ErasureCodingSmallBlockSize)
+ }
+ }
+
tmpBase := base + ".ecxrecover"
tmpDat := tmpBase + ".dat"
- if err := erasure_coding.WriteDatFile(tmpBase, reconstructSize, reconstructSize, shardFileNames); err != nil {
- os.Remove(tmpDat)
- fail(fmt.Errorf("volume %d: reconstruct .dat from data shards: %w", volumeId, err))
+ defer os.Remove(tmpDat)
+ bestIdx := -1
+ var realDatSize, bestNeedles int64
+ var version needle.Version
+ var candErr error
+ for i, cand := range candidates {
+ if err := erasure_coding.WriteDatFile(tmpBase, reconstructSize, reconstructSize, shardFileNames, cand.large, cand.small); err != nil {
+ candErr = fmt.Errorf("volume %d: reconstruct .dat (%s layout): %w", volumeId, cand.name, err)
+ continue
+ }
+ size, needles, ver, err := writeEcxFromDat(tmpDat, ecxName+"."+cand.name)
+ if err != nil {
+ os.Remove(ecxName + "." + cand.name)
+ candErr = fmt.Errorf("volume %d: build .ecx from reconstructed .dat (%s layout): %w", volumeId, cand.name, err)
+ continue
+ }
+ // The wrong layout de-stripes to garbage past the first block
+ // boundary, so the layout that indexes more valid needles wins.
+ // On a tie — garbage bytes do sometimes parse as plausible sizes — the
+ // layout whose needle chain reached further into the .dat wins, which is
+ // the distance the scan actually validated.
+ if bestIdx < 0 || needles > bestNeedles || (needles == bestNeedles && size > realDatSize) {
+ bestIdx = i
+ bestNeedles = needles
+ realDatSize = size
+ version = ver
+ }
+ }
+ if bestIdx < 0 {
+ fail(candErr)
return
}
- defer os.Remove(tmpDat)
-
- realDatSize, version, err := writeEcxFromDat(tmpDat, ecxName)
- if err != nil {
- os.Remove(ecxName)
- fail(fmt.Errorf("volume %d: build .ecx from reconstructed .dat: %w", volumeId, err))
+ for i := range candidates {
+ if i != bestIdx {
+ os.Remove(ecxName + "." + candidates[i].name)
+ }
+ }
+ if err := os.Rename(ecxName+"."+candidates[bestIdx].name, ecxName); err != nil {
+ fail(fmt.Errorf("volume %d: publish %s: %w", volumeId, ecxName, err))
return
}
+ if blockSize < 0 {
+ if candidates[bestIdx].name == "uniform" {
+ blockSize = shardSize
+ } else {
+ blockSize = 0
+ }
+ glog.Infof("volume %d: %s layout indexes %d needles over %d bytes; keeping it", volumeId, candidates[bestIdx].name, bestNeedles, realDatSize)
+ }
glog.Infof("volume %d: wrote %s from %d data shards", volumeId, ecxName, dataShards)
- // Regenerate the .vif when missing so the volume can mount and future
- // rebuilds know the EC ratio and original .dat size.
- if !vifExists {
+ // Regenerate the .vif when it is missing OR unusable (an empty stub, or one
+ // with no EC config): the layout just recovered by the dual scan is the only
+ // record of it, and leaving the stub in place would mount the volume as
+ // legacy on the next start and serve the wrong offsets.
+ if !vifUsable {
size := datFileSize
if size <= 0 {
size = realDatSize
}
+ // Never publish a layout the mount path will refuse: NewEcVolume runs
+ // the same check and fails closed, so an unvalidated write here would
+ // trade a recoverable volume for one that can no longer be opened.
+ if bsErr := erasure_coding.ValidateBlockSize(blockSize); bsErr != nil {
+ fail(fmt.Errorf("volume %d: refusing to write %s: %w", volumeId, vifName, bsErr))
+ return
+ }
volumeInfo := &volume_server_pb.VolumeInfo{
Version: uint32(version),
DatFileSize: size,
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: uint32(dataShards),
ParityShards: uint32(parityShards),
+ BlockSize: blockSize,
},
}
if err := volume_info.SaveVolumeInfo(vifName, volumeInfo); err != nil {
@@ -451,25 +590,26 @@ func doFixEcxFromShards(basePath, baseFileName, collection string, volumeId int6
// writeEcxFromDat scans a (reconstructed) .dat and writes an ascending-sorted
// .ecx containing only live needles — the same on-disk shape
// WriteSortedFileFromIdx produces when an EC volume is first encoded. It returns
-// the physical .dat size (the offset where the EC zero padding begins) and the
-// volume version read from the superblock.
-func writeEcxFromDat(datPath, ecxPath string) (datFileSize int64, version needle.Version, err error) {
+// the physical .dat size (the offset where the EC zero padding begins), the
+// number of live needles indexed, and the volume version read from the
+// superblock.
+func writeEcxFromDat(datPath, ecxPath string) (datFileSize int64, liveNeedles int64, version needle.Version, err error) {
f, err := os.OpenFile(datPath, os.O_RDONLY, 0644)
if err != nil {
- return 0, 0, fmt.Errorf("open %s: %w", datPath, err)
+ return 0, 0, 0, fmt.Errorf("open %s: %w", datPath, err)
}
datBackend := backend.NewDiskFile(f)
defer datBackend.Close()
superBlock, err := super_block.ReadSuperBlock(datBackend)
if err != nil {
- return 0, 0, fmt.Errorf("read superblock: %w", err)
+ return 0, 0, 0, fmt.Errorf("read superblock: %w", err)
}
version = superBlock.Version
fileSize, _, err := datBackend.GetStat()
if err != nil {
- return 0, version, fmt.Errorf("stat %s: %w", datPath, err)
+ return 0, 0, version, fmt.Errorf("stat %s: %w", datPath, err)
}
nm := needle_map.NewMemDb()
@@ -482,7 +622,7 @@ func writeEcxFromDat(datPath, ecxPath string) (datFileSize int64, version needle
if readErr == io.EOF {
break
}
- return 0, version, fmt.Errorf("read needle header at offset %d: %w", offset, readErr)
+ return 0, 0, version, fmt.Errorf("read needle header at offset %d: %w", offset, readErr)
}
// EC encoding zero-pads the tail of the last block row. An all-zero
// header marks the start of that padding, i.e. the end of real needles.
@@ -491,13 +631,13 @@ func writeEcxFromDat(datPath, ecxPath string) (datFileSize int64, version needle
}
if n.Size.IsValid() {
if pe := nm.Set(n.Id, types.ToOffset(offset), n.Size); pe != nil {
- return 0, version, fmt.Errorf("set needle %d: %w", n.Id, pe)
+ return 0, 0, version, fmt.Errorf("set needle %d: %w", n.Id, pe)
}
} else {
// Deleted/invalid: drop it so the .ecx carries only live entries,
// matching the encode-time WriteSortedFileFromIdx behavior.
if pe := nm.Delete(n.Id); pe != nil {
- return 0, version, fmt.Errorf("delete needle %d: %w", n.Id, pe)
+ return 0, 0, version, fmt.Errorf("delete needle %d: %w", n.Id, pe)
}
}
offset += types.NeedleHeaderSize + rest
@@ -506,16 +646,17 @@ func writeEcxFromDat(datPath, ecxPath string) (datFileSize int64, version needle
ecxFile, err := os.OpenFile(ecxPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
- return 0, version, fmt.Errorf("open %s: %w", ecxPath, err)
+ return 0, 0, version, fmt.Errorf("open %s: %w", ecxPath, err)
}
defer ecxFile.Close()
if err := nm.AscendingVisit(func(value needle_map.NeedleValue) error {
+ liveNeedles++
_, writeErr := ecxFile.Write(value.ToBytes())
return writeErr
}); err != nil {
- return 0, version, fmt.Errorf("write %s: %w", ecxPath, err)
+ return 0, 0, version, fmt.Errorf("write %s: %w", ecxPath, err)
}
- return datFileSize, version, nil
+ return datFileSize, liveNeedles, version, nil
}
diff --git a/weed/command/fix_ecx_test.go b/weed/command/fix_ecx_test.go
index d99f826945..c9509b0eb1 100644
--- a/weed/command/fix_ecx_test.go
+++ b/weed/command/fix_ecx_test.go
@@ -17,11 +17,12 @@ import (
"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
)
-// buildAndEncodeTestEcVolume writes a small volume (.dat + .idx), EC-encodes it
-// into .ec00..ec13, and produces the canonical sorted .ecx. It returns the base
-// path, the canonical .ecx bytes, and the original .dat size. A couple of
-// needles are deleted so the .ecx must exclude them (live entries only).
-func buildAndEncodeTestEcVolume(t *testing.T, dir, baseName string) (base string, canonicalEcx []byte, origDatSize int64) {
+// buildAndEncodeTestEcVolume writes a volume of needleCount needles of about
+// needleSize bytes (.dat + .idx), EC-encodes it into .ec00..ec13, and produces
+// the canonical sorted .ecx. It returns the base path, the canonical .ecx
+// bytes, the original .dat size, and the encode's bitrot protection. A couple
+// of needles are deleted so the .ecx must exclude them (live entries only).
+func buildAndEncodeTestEcVolume(t *testing.T, dir, baseName string, needleCount, needleSize int) (base string, canonicalEcx []byte, origDatSize int64, prot *volume_server_pb.EcBitrotProtection) {
t.Helper()
base = filepath.Join(dir, baseName)
version := needle.GetCurrentVersion()
@@ -41,10 +42,10 @@ func buildAndEncodeTestEcVolume(t *testing.T, dir, baseName string) (base string
}
nm := needle_map.NewMemDb()
- for i := uint64(1); i <= 12; i++ {
+ for i := uint64(1); i <= uint64(needleCount); i++ {
n := new(needle.Needle)
n.Id = types.Uint64ToNeedleId(i)
- n.Data = make([]byte, 200+int(i))
+ n.Data = make([]byte, needleSize+int(i))
rand.Read(n.Data)
n.Checksum = needle.NewCRC(n.Data)
offset, _, _, err := n.Append(datBackend, version)
@@ -92,7 +93,8 @@ func buildAndEncodeTestEcVolume(t *testing.T, dir, baseName string) (base string
idxFile.Close()
nm.Close()
- if _, err := erasure_coding.WriteEcFiles(base, erasure_coding.BackgroundECContext()); err != nil {
+ prot, err = erasure_coding.WriteEcFiles(base, erasure_coding.BackgroundECContext())
+ if err != nil {
t.Fatalf("WriteEcFiles: %v", err)
}
if err := erasure_coding.WriteSortedFileFromIdx(base, ".ecx"); err != nil {
@@ -105,7 +107,7 @@ func buildAndEncodeTestEcVolume(t *testing.T, dir, baseName string) (base string
if len(canonicalEcx) == 0 {
t.Fatal("canonical .ecx is empty")
}
- return base, canonicalEcx, origDatSize
+ return base, canonicalEcx, origDatSize, prot
}
// TestFixEcxFromShards verifies the .ecx and .vif are rebuilt purely from the
@@ -121,7 +123,7 @@ func TestFixEcxFromShards(t *testing.T) {
dir := t.TempDir()
const volumeId = 7
- base, canonical, origDatSize := buildAndEncodeTestEcVolume(t, dir, "7")
+ base, canonical, origDatSize, _ := buildAndEncodeTestEcVolume(t, dir, "7", 12, 200)
// Disaster: keep only the shards.
for _, ext := range []string{".ecx", ".ecj", ".idx", ".dat", ".vif"} {
@@ -175,7 +177,7 @@ func TestFixEcxFromShardsWithVif(t *testing.T) {
dir := t.TempDir()
const volumeId = 9
- base, canonical, origDatSize := buildAndEncodeTestEcVolume(t, dir, "9")
+ base, canonical, origDatSize, _ := buildAndEncodeTestEcVolume(t, dir, "9", 12, 200)
// Write a .vif as the volume server would after encoding.
if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
@@ -234,7 +236,7 @@ func TestFixEcxFromShardsMissingShards(t *testing.T) {
dir := t.TempDir()
const volumeId = 11
- base, canonical, origDatSize := buildAndEncodeTestEcVolume(t, dir, "11")
+ base, canonical, origDatSize, _ := buildAndEncodeTestEcVolume(t, dir, "11", 12, 200)
// Lose every index/metadata file plus three shards (two data: .ec02, .ec05;
// one parity: .ec11), keeping 11 of 14 — enough to reconstruct. The highest
@@ -271,3 +273,60 @@ func TestFixEcxFromShardsMissingShards(t *testing.T) {
t.Fatalf("dat size = %d, want %d", vi.GetDatFileSize(), origDatSize)
}
}
+
+// TestFixEcxFromShardsUniformLayout loses the .vif of a volume big enough that
+// the uniform and legacy layouts place bytes differently, and verifies the
+// layout is recovered — from the bitrot sidecar when it survives, and by
+// de-striping under both candidate layouts when it does not.
+func TestFixEcxFromShardsUniformLayout(t *testing.T) {
+ oldData, oldParity := *fixEcDataShards, *fixEcParityShards
+ *fixEcDataShards, *fixEcParityShards = 0, 0
+ *fixIgnoreError = true
+ t.Cleanup(func() {
+ *fixIgnoreError = false
+ *fixEcDataShards, *fixEcParityShards = oldData, oldParity
+ })
+
+ for _, withSidecar := range []bool{true, false} {
+ name := "from-sidecar"
+ if !withSidecar {
+ name = "dual-scan"
+ }
+ t.Run(name, func(t *testing.T) {
+ dir := t.TempDir()
+ const volumeId = 13
+ base, canonical, _, prot := buildAndEncodeTestEcVolume(t, dir, "13", 13, 2*1024*1024)
+ if got := prot.GetEcShardConfig().GetBlockSize(); got <= erasure_coding.ErasureCodingSmallBlockSize {
+ t.Fatalf("fixture block size %d does not diverge from the legacy layout", got)
+ }
+ if withSidecar {
+ if err := erasure_coding.SaveBitrotSidecar(erasure_coding.BitrotSidecarPath(base, 0), prot); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ for _, ext := range []string{".ecx", ".ecj", ".idx", ".dat", ".vif"} {
+ if err := os.Remove(base + ext); err != nil && !os.IsNotExist(err) {
+ t.Fatalf("remove %s: %v", base+ext, err)
+ }
+ }
+
+ doFixEcxFromShards(dir, "13", "", volumeId)
+
+ recovered, err := os.ReadFile(base + ".ecx")
+ if err != nil {
+ t.Fatalf("recovered .ecx not written: %v", err)
+ }
+ if !bytes.Equal(canonical, recovered) {
+ t.Fatalf(".ecx mismatch: canonical %d bytes, recovered %d bytes", len(canonical), len(recovered))
+ }
+ vi, _, found, err := volume_info.MaybeLoadVolumeInfo(base + ".vif")
+ if err != nil || !found {
+ t.Fatalf(".vif not regenerated: found=%v err=%v", found, err)
+ }
+ if got, want := vi.GetEcShardConfig().GetBlockSize(), prot.GetEcShardConfig().GetBlockSize(); got != want {
+ t.Fatalf("regenerated .vif block size = %d, want %d", got, want)
+ }
+ })
+ }
+}
diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto
index d44712c3f3..4962a4e9f6 100644
--- a/weed/pb/volume_server.proto
+++ b/weed/pb/volume_server.proto
@@ -541,6 +541,7 @@ message VolumeEcShardsInfoResponse {
uint64 volume_size = 2;
uint64 file_count = 3;
uint64 file_deleted_count = 4;
+ EcShardConfig ec_shard_config = 10; // the layout this holder serves reads through; a binary predating a field reports it as unset
}
message EcShardInfo {
@@ -615,6 +616,7 @@ message EcShardConfig {
uint32 data_shards = 1; // Number of data shards (e.g., 10)
uint32 parity_shards = 2; // Number of parity shards (e.g., 4)
int64 encode_ts_ns = 3; // encode time (unix nanos); a read served from a shard of a different encode run is rejected
+ int64 block_size = 4; // uniform block layout: each shard is a single contiguous block of this many bytes; 0 = legacy 1GiB/1MiB two-tier layout
}
// EcBitrotProtection is the entire content of a bitrot checksum sidecar
diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go
index afe59bc085..24ad977788 100644
--- a/weed/pb/volume_server_pb/volume_server.pb.go
+++ b/weed/pb/volume_server_pb/volume_server.pb.go
@@ -4448,6 +4448,7 @@ type VolumeEcShardsInfoResponse struct {
VolumeSize uint64 `protobuf:"varint,2,opt,name=volume_size,json=volumeSize,proto3" json:"volume_size,omitempty"`
FileCount uint64 `protobuf:"varint,3,opt,name=file_count,json=fileCount,proto3" json:"file_count,omitempty"`
FileDeletedCount uint64 `protobuf:"varint,4,opt,name=file_deleted_count,json=fileDeletedCount,proto3" json:"file_deleted_count,omitempty"`
+ EcShardConfig *EcShardConfig `protobuf:"bytes,10,opt,name=ec_shard_config,json=ecShardConfig,proto3" json:"ec_shard_config,omitempty"` // the layout this holder serves reads through; a binary predating a field reports it as unset
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -4510,6 +4511,13 @@ func (x *VolumeEcShardsInfoResponse) GetFileDeletedCount() uint64 {
return 0
}
+func (x *VolumeEcShardsInfoResponse) GetEcShardConfig() *EcShardConfig {
+ if x != nil {
+ return x.EcShardConfig
+ }
+ return nil
+}
+
type EcShardInfo struct {
state protoimpl.MessageState `protogen:"open.v1"`
ShardId uint32 `protobuf:"varint,1,opt,name=shard_id,json=shardId,proto3" json:"shard_id,omitempty"`
@@ -5145,6 +5153,7 @@ type EcShardConfig struct {
DataShards uint32 `protobuf:"varint,1,opt,name=data_shards,json=dataShards,proto3" json:"data_shards,omitempty"` // Number of data shards (e.g., 10)
ParityShards uint32 `protobuf:"varint,2,opt,name=parity_shards,json=parityShards,proto3" json:"parity_shards,omitempty"` // Number of parity shards (e.g., 4)
EncodeTsNs int64 `protobuf:"varint,3,opt,name=encode_ts_ns,json=encodeTsNs,proto3" json:"encode_ts_ns,omitempty"` // encode time (unix nanos); a read served from a shard of a different encode run is rejected
+ BlockSize int64 `protobuf:"varint,4,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` // uniform block layout: each shard is a single contiguous block of this many bytes; 0 = legacy 1GiB/1MiB two-tier layout
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -5200,6 +5209,13 @@ func (x *EcShardConfig) GetEncodeTsNs() int64 {
return 0
}
+func (x *EcShardConfig) GetBlockSize() int64 {
+ if x != nil {
+ return x.BlockSize
+ }
+ return 0
+}
+
// EcBitrotProtection is the entire content of a bitrot checksum sidecar
// (<base>.ecsum for the legacy generation, <base>.ecsum.v<N> for vacuum
// generation N). On disk it is wrapped in a fixed header carrying a CRC32C
@@ -7509,14 +7525,16 @@ const file_volume_server_proto_rawDesc = "" +
"\tdisk_type\x18\x04 \x01(\tR\bdiskType\" \n" +
"\x1eVolumeEcShardsToVolumeResponse\"8\n" +
"\x19VolumeEcShardsInfoRequest\x12\x1b\n" +
- "\tvolume_id\x18\x01 \x01(\rR\bvolumeId\"\xcf\x01\n" +
+ "\tvolume_id\x18\x01 \x01(\rR\bvolumeId\"\x98\x02\n" +
"\x1aVolumeEcShardsInfoResponse\x12C\n" +
"\x0eec_shard_infos\x18\x01 \x03(\v2\x1d.volume_server_pb.EcShardInfoR\fecShardInfos\x12\x1f\n" +
"\vvolume_size\x18\x02 \x01(\x04R\n" +
"volumeSize\x12\x1d\n" +
"\n" +
"file_count\x18\x03 \x01(\x04R\tfileCount\x12,\n" +
- "\x12file_deleted_count\x18\x04 \x01(\x04R\x10fileDeletedCount\"y\n" +
+ "\x12file_deleted_count\x18\x04 \x01(\x04R\x10fileDeletedCount\x12G\n" +
+ "\x0fec_shard_config\x18\n" +
+ " \x01(\v2\x1f.volume_server_pb.EcShardConfigR\recShardConfig\"y\n" +
"\vEcShardInfo\x12\x19\n" +
"\bshard_id\x18\x01 \x01(\rR\ashardId\x12\x12\n" +
"\x04size\x18\x02 \x01(\x03R\x04size\x12\x1e\n" +
@@ -7583,13 +7601,15 @@ const file_volume_server_proto_rawDesc = "" +
"\rexpire_at_sec\x18\x06 \x01(\x04R\vexpireAtSec\x12\x1b\n" +
"\tread_only\x18\a \x01(\bR\breadOnly\x12G\n" +
"\x0fec_shard_config\x18\b \x01(\v2\x1f.volume_server_pb.EcShardConfigR\recShardConfig\x12/\n" +
- "\x14read_only_can_delete\x18\t \x01(\bR\x11readOnlyCanDelete\"w\n" +
+ "\x14read_only_can_delete\x18\t \x01(\bR\x11readOnlyCanDelete\"\x96\x01\n" +
"\rEcShardConfig\x12\x1f\n" +
"\vdata_shards\x18\x01 \x01(\rR\n" +
"dataShards\x12#\n" +
"\rparity_shards\x18\x02 \x01(\rR\fparityShards\x12 \n" +
"\fencode_ts_ns\x18\x03 \x01(\x03R\n" +
- "encodeTsNs\"\xbc\x02\n" +
+ "encodeTsNs\x12\x1d\n" +
+ "\n" +
+ "block_size\x18\x04 \x01(\x03R\tblockSize\"\xbc\x02\n" +
"\x12EcBitrotProtection\x12A\n" +
"\talgorithm\x18\x01 \x01(\x0e2#.volume_server_pb.ChecksumAlgorithmR\talgorithm\x12\x1d\n" +
"\n" +
@@ -7958,133 +7978,134 @@ var file_volume_server_proto_depIdxs = []int32{
2, // 3: volume_server_pb.SetStateResponse.state:type_name -> volume_server_pb.VolumeServerState
48, // 4: volume_server_pb.ReceiveFileRequest.info:type_name -> volume_server_pb.ReceiveFileInfo
82, // 5: volume_server_pb.VolumeEcShardsInfoResponse.ec_shard_infos:type_name -> volume_server_pb.EcShardInfo
- 88, // 6: volume_server_pb.ReadVolumeFileStatusResponse.volume_info:type_name -> volume_server_pb.VolumeInfo
- 87, // 7: volume_server_pb.VolumeInfo.files:type_name -> volume_server_pb.RemoteFile
- 89, // 8: volume_server_pb.VolumeInfo.ec_shard_config:type_name -> volume_server_pb.EcShardConfig
- 0, // 9: volume_server_pb.EcBitrotProtection.algorithm:type_name -> volume_server_pb.ChecksumAlgorithm
- 89, // 10: volume_server_pb.EcBitrotProtection.ec_shard_config:type_name -> volume_server_pb.EcShardConfig
- 91, // 11: volume_server_pb.EcBitrotProtection.shards:type_name -> volume_server_pb.EcShardChecksums
- 87, // 12: volume_server_pb.OldVersionVolumeInfo.files:type_name -> volume_server_pb.RemoteFile
- 85, // 13: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus
- 86, // 14: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus
- 2, // 15: volume_server_pb.VolumeServerStatusResponse.state:type_name -> volume_server_pb.VolumeServerState
- 113, // 16: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica
- 122, // 17: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf
- 123, // 18: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation
- 1, // 19: volume_server_pb.ScrubVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
- 1, // 20: volume_server_pb.ScrubEcVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
- 82, // 21: volume_server_pb.ScrubEcVolumeResponse.broken_shard_infos:type_name -> volume_server_pb.EcShardInfo
- 114, // 22: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter
- 115, // 23: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization
- 116, // 24: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization
- 117, // 25: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput
- 118, // 26: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput
- 119, // 27: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput
- 120, // 28: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
- 121, // 29: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
- 3, // 30: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest
- 7, // 31: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest
- 9, // 32: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest
- 11, // 33: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest
- 13, // 34: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest
- 15, // 35: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest
- 17, // 36: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest
- 19, // 37: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest
- 21, // 38: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest
- 23, // 39: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest
- 25, // 40: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest
- 27, // 41: volume_server_pb.VolumeServer.VolumeConsolidateIndex:input_type -> volume_server_pb.VolumeConsolidateIndexRequest
- 29, // 42: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest
- 31, // 43: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest
- 33, // 44: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest
- 35, // 45: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest
- 37, // 46: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest
- 39, // 47: volume_server_pb.VolumeServer.GetState:input_type -> volume_server_pb.GetStateRequest
- 41, // 48: volume_server_pb.VolumeServer.SetState:input_type -> volume_server_pb.SetStateRequest
- 43, // 49: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest
- 83, // 50: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest
- 45, // 51: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest
- 47, // 52: volume_server_pb.VolumeServer.ReceiveFile:input_type -> volume_server_pb.ReceiveFileRequest
- 50, // 53: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest
- 52, // 54: volume_server_pb.VolumeServer.ReadNeedleMeta:input_type -> volume_server_pb.ReadNeedleMetaRequest
- 54, // 55: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest
- 56, // 56: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest
- 58, // 57: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest
- 60, // 58: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest
- 62, // 59: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest
- 64, // 60: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest
- 66, // 61: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest
- 68, // 62: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest
- 70, // 63: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest
- 72, // 64: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest
- 74, // 65: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest
- 76, // 66: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest
- 78, // 67: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest
- 80, // 68: volume_server_pb.VolumeServer.VolumeEcShardsInfo:input_type -> volume_server_pb.VolumeEcShardsInfoRequest
- 93, // 69: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest
- 95, // 70: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest
- 97, // 71: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest
- 99, // 72: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest
- 101, // 73: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest
- 103, // 74: volume_server_pb.VolumeServer.ScrubVolume:input_type -> volume_server_pb.ScrubVolumeRequest
- 105, // 75: volume_server_pb.VolumeServer.ScrubEcVolume:input_type -> volume_server_pb.ScrubEcVolumeRequest
- 107, // 76: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest
- 109, // 77: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest
- 111, // 78: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest
- 4, // 79: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse
- 8, // 80: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse
- 10, // 81: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse
- 12, // 82: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse
- 14, // 83: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse
- 16, // 84: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse
- 18, // 85: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse
- 20, // 86: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse
- 22, // 87: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse
- 24, // 88: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse
- 26, // 89: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse
- 28, // 90: volume_server_pb.VolumeServer.VolumeConsolidateIndex:output_type -> volume_server_pb.VolumeConsolidateIndexResponse
- 30, // 91: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse
- 32, // 92: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse
- 34, // 93: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse
- 36, // 94: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse
- 38, // 95: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse
- 40, // 96: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse
- 42, // 97: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse
- 44, // 98: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse
- 84, // 99: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse
- 46, // 100: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse
- 49, // 101: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse
- 51, // 102: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse
- 53, // 103: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse
- 55, // 104: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse
- 57, // 105: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse
- 59, // 106: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse
- 61, // 107: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse
- 63, // 108: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse
- 65, // 109: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse
- 67, // 110: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse
- 69, // 111: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse
- 71, // 112: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse
- 73, // 113: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse
- 75, // 114: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse
- 77, // 115: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse
- 79, // 116: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse
- 81, // 117: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse
- 94, // 118: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse
- 96, // 119: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse
- 98, // 120: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse
- 100, // 121: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse
- 102, // 122: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse
- 104, // 123: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse
- 106, // 124: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse
- 108, // 125: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe
- 110, // 126: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse
- 112, // 127: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse
- 79, // [79:128] is the sub-list for method output_type
- 30, // [30:79] is the sub-list for method input_type
- 30, // [30:30] is the sub-list for extension type_name
- 30, // [30:30] is the sub-list for extension extendee
- 0, // [0:30] is the sub-list for field type_name
+ 89, // 6: volume_server_pb.VolumeEcShardsInfoResponse.ec_shard_config:type_name -> volume_server_pb.EcShardConfig
+ 88, // 7: volume_server_pb.ReadVolumeFileStatusResponse.volume_info:type_name -> volume_server_pb.VolumeInfo
+ 87, // 8: volume_server_pb.VolumeInfo.files:type_name -> volume_server_pb.RemoteFile
+ 89, // 9: volume_server_pb.VolumeInfo.ec_shard_config:type_name -> volume_server_pb.EcShardConfig
+ 0, // 10: volume_server_pb.EcBitrotProtection.algorithm:type_name -> volume_server_pb.ChecksumAlgorithm
+ 89, // 11: volume_server_pb.EcBitrotProtection.ec_shard_config:type_name -> volume_server_pb.EcShardConfig
+ 91, // 12: volume_server_pb.EcBitrotProtection.shards:type_name -> volume_server_pb.EcShardChecksums
+ 87, // 13: volume_server_pb.OldVersionVolumeInfo.files:type_name -> volume_server_pb.RemoteFile
+ 85, // 14: volume_server_pb.VolumeServerStatusResponse.disk_statuses:type_name -> volume_server_pb.DiskStatus
+ 86, // 15: volume_server_pb.VolumeServerStatusResponse.memory_status:type_name -> volume_server_pb.MemStatus
+ 2, // 16: volume_server_pb.VolumeServerStatusResponse.state:type_name -> volume_server_pb.VolumeServerState
+ 113, // 17: volume_server_pb.FetchAndWriteNeedleRequest.replicas:type_name -> volume_server_pb.FetchAndWriteNeedleRequest.Replica
+ 122, // 18: volume_server_pb.FetchAndWriteNeedleRequest.remote_conf:type_name -> remote_pb.RemoteConf
+ 123, // 19: volume_server_pb.FetchAndWriteNeedleRequest.remote_location:type_name -> remote_pb.RemoteStorageLocation
+ 1, // 20: volume_server_pb.ScrubVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
+ 1, // 21: volume_server_pb.ScrubEcVolumeRequest.mode:type_name -> volume_server_pb.VolumeScrubMode
+ 82, // 22: volume_server_pb.ScrubEcVolumeResponse.broken_shard_infos:type_name -> volume_server_pb.EcShardInfo
+ 114, // 23: volume_server_pb.QueryRequest.filter:type_name -> volume_server_pb.QueryRequest.Filter
+ 115, // 24: volume_server_pb.QueryRequest.input_serialization:type_name -> volume_server_pb.QueryRequest.InputSerialization
+ 116, // 25: volume_server_pb.QueryRequest.output_serialization:type_name -> volume_server_pb.QueryRequest.OutputSerialization
+ 117, // 26: volume_server_pb.QueryRequest.InputSerialization.csv_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.CSVInput
+ 118, // 27: volume_server_pb.QueryRequest.InputSerialization.json_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.JSONInput
+ 119, // 28: volume_server_pb.QueryRequest.InputSerialization.parquet_input:type_name -> volume_server_pb.QueryRequest.InputSerialization.ParquetInput
+ 120, // 29: volume_server_pb.QueryRequest.OutputSerialization.csv_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.CSVOutput
+ 121, // 30: volume_server_pb.QueryRequest.OutputSerialization.json_output:type_name -> volume_server_pb.QueryRequest.OutputSerialization.JSONOutput
+ 3, // 31: volume_server_pb.VolumeServer.BatchDelete:input_type -> volume_server_pb.BatchDeleteRequest
+ 7, // 32: volume_server_pb.VolumeServer.VacuumVolumeCheck:input_type -> volume_server_pb.VacuumVolumeCheckRequest
+ 9, // 33: volume_server_pb.VolumeServer.VacuumVolumeCompact:input_type -> volume_server_pb.VacuumVolumeCompactRequest
+ 11, // 34: volume_server_pb.VolumeServer.VacuumVolumeCommit:input_type -> volume_server_pb.VacuumVolumeCommitRequest
+ 13, // 35: volume_server_pb.VolumeServer.VacuumVolumeCleanup:input_type -> volume_server_pb.VacuumVolumeCleanupRequest
+ 15, // 36: volume_server_pb.VolumeServer.DeleteCollection:input_type -> volume_server_pb.DeleteCollectionRequest
+ 17, // 37: volume_server_pb.VolumeServer.AllocateVolume:input_type -> volume_server_pb.AllocateVolumeRequest
+ 19, // 38: volume_server_pb.VolumeServer.VolumeSyncStatus:input_type -> volume_server_pb.VolumeSyncStatusRequest
+ 21, // 39: volume_server_pb.VolumeServer.VolumeIncrementalCopy:input_type -> volume_server_pb.VolumeIncrementalCopyRequest
+ 23, // 40: volume_server_pb.VolumeServer.VolumeMount:input_type -> volume_server_pb.VolumeMountRequest
+ 25, // 41: volume_server_pb.VolumeServer.VolumeUnmount:input_type -> volume_server_pb.VolumeUnmountRequest
+ 27, // 42: volume_server_pb.VolumeServer.VolumeConsolidateIndex:input_type -> volume_server_pb.VolumeConsolidateIndexRequest
+ 29, // 43: volume_server_pb.VolumeServer.VolumeDelete:input_type -> volume_server_pb.VolumeDeleteRequest
+ 31, // 44: volume_server_pb.VolumeServer.VolumeMarkReadonly:input_type -> volume_server_pb.VolumeMarkReadonlyRequest
+ 33, // 45: volume_server_pb.VolumeServer.VolumeMarkWritable:input_type -> volume_server_pb.VolumeMarkWritableRequest
+ 35, // 46: volume_server_pb.VolumeServer.VolumeConfigure:input_type -> volume_server_pb.VolumeConfigureRequest
+ 37, // 47: volume_server_pb.VolumeServer.VolumeStatus:input_type -> volume_server_pb.VolumeStatusRequest
+ 39, // 48: volume_server_pb.VolumeServer.GetState:input_type -> volume_server_pb.GetStateRequest
+ 41, // 49: volume_server_pb.VolumeServer.SetState:input_type -> volume_server_pb.SetStateRequest
+ 43, // 50: volume_server_pb.VolumeServer.VolumeCopy:input_type -> volume_server_pb.VolumeCopyRequest
+ 83, // 51: volume_server_pb.VolumeServer.ReadVolumeFileStatus:input_type -> volume_server_pb.ReadVolumeFileStatusRequest
+ 45, // 52: volume_server_pb.VolumeServer.CopyFile:input_type -> volume_server_pb.CopyFileRequest
+ 47, // 53: volume_server_pb.VolumeServer.ReceiveFile:input_type -> volume_server_pb.ReceiveFileRequest
+ 50, // 54: volume_server_pb.VolumeServer.ReadNeedleBlob:input_type -> volume_server_pb.ReadNeedleBlobRequest
+ 52, // 55: volume_server_pb.VolumeServer.ReadNeedleMeta:input_type -> volume_server_pb.ReadNeedleMetaRequest
+ 54, // 56: volume_server_pb.VolumeServer.WriteNeedleBlob:input_type -> volume_server_pb.WriteNeedleBlobRequest
+ 56, // 57: volume_server_pb.VolumeServer.ReadAllNeedles:input_type -> volume_server_pb.ReadAllNeedlesRequest
+ 58, // 58: volume_server_pb.VolumeServer.VolumeTailSender:input_type -> volume_server_pb.VolumeTailSenderRequest
+ 60, // 59: volume_server_pb.VolumeServer.VolumeTailReceiver:input_type -> volume_server_pb.VolumeTailReceiverRequest
+ 62, // 60: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:input_type -> volume_server_pb.VolumeEcShardsGenerateRequest
+ 64, // 61: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:input_type -> volume_server_pb.VolumeEcShardsRebuildRequest
+ 66, // 62: volume_server_pb.VolumeServer.VolumeEcShardsCopy:input_type -> volume_server_pb.VolumeEcShardsCopyRequest
+ 68, // 63: volume_server_pb.VolumeServer.VolumeEcShardsDelete:input_type -> volume_server_pb.VolumeEcShardsDeleteRequest
+ 70, // 64: volume_server_pb.VolumeServer.VolumeEcShardsMount:input_type -> volume_server_pb.VolumeEcShardsMountRequest
+ 72, // 65: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:input_type -> volume_server_pb.VolumeEcShardsUnmountRequest
+ 74, // 66: volume_server_pb.VolumeServer.VolumeEcShardRead:input_type -> volume_server_pb.VolumeEcShardReadRequest
+ 76, // 67: volume_server_pb.VolumeServer.VolumeEcBlobDelete:input_type -> volume_server_pb.VolumeEcBlobDeleteRequest
+ 78, // 68: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:input_type -> volume_server_pb.VolumeEcShardsToVolumeRequest
+ 80, // 69: volume_server_pb.VolumeServer.VolumeEcShardsInfo:input_type -> volume_server_pb.VolumeEcShardsInfoRequest
+ 93, // 70: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:input_type -> volume_server_pb.VolumeTierMoveDatToRemoteRequest
+ 95, // 71: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:input_type -> volume_server_pb.VolumeTierMoveDatFromRemoteRequest
+ 97, // 72: volume_server_pb.VolumeServer.VolumeServerStatus:input_type -> volume_server_pb.VolumeServerStatusRequest
+ 99, // 73: volume_server_pb.VolumeServer.VolumeServerLeave:input_type -> volume_server_pb.VolumeServerLeaveRequest
+ 101, // 74: volume_server_pb.VolumeServer.FetchAndWriteNeedle:input_type -> volume_server_pb.FetchAndWriteNeedleRequest
+ 103, // 75: volume_server_pb.VolumeServer.ScrubVolume:input_type -> volume_server_pb.ScrubVolumeRequest
+ 105, // 76: volume_server_pb.VolumeServer.ScrubEcVolume:input_type -> volume_server_pb.ScrubEcVolumeRequest
+ 107, // 77: volume_server_pb.VolumeServer.Query:input_type -> volume_server_pb.QueryRequest
+ 109, // 78: volume_server_pb.VolumeServer.VolumeNeedleStatus:input_type -> volume_server_pb.VolumeNeedleStatusRequest
+ 111, // 79: volume_server_pb.VolumeServer.Ping:input_type -> volume_server_pb.PingRequest
+ 4, // 80: volume_server_pb.VolumeServer.BatchDelete:output_type -> volume_server_pb.BatchDeleteResponse
+ 8, // 81: volume_server_pb.VolumeServer.VacuumVolumeCheck:output_type -> volume_server_pb.VacuumVolumeCheckResponse
+ 10, // 82: volume_server_pb.VolumeServer.VacuumVolumeCompact:output_type -> volume_server_pb.VacuumVolumeCompactResponse
+ 12, // 83: volume_server_pb.VolumeServer.VacuumVolumeCommit:output_type -> volume_server_pb.VacuumVolumeCommitResponse
+ 14, // 84: volume_server_pb.VolumeServer.VacuumVolumeCleanup:output_type -> volume_server_pb.VacuumVolumeCleanupResponse
+ 16, // 85: volume_server_pb.VolumeServer.DeleteCollection:output_type -> volume_server_pb.DeleteCollectionResponse
+ 18, // 86: volume_server_pb.VolumeServer.AllocateVolume:output_type -> volume_server_pb.AllocateVolumeResponse
+ 20, // 87: volume_server_pb.VolumeServer.VolumeSyncStatus:output_type -> volume_server_pb.VolumeSyncStatusResponse
+ 22, // 88: volume_server_pb.VolumeServer.VolumeIncrementalCopy:output_type -> volume_server_pb.VolumeIncrementalCopyResponse
+ 24, // 89: volume_server_pb.VolumeServer.VolumeMount:output_type -> volume_server_pb.VolumeMountResponse
+ 26, // 90: volume_server_pb.VolumeServer.VolumeUnmount:output_type -> volume_server_pb.VolumeUnmountResponse
+ 28, // 91: volume_server_pb.VolumeServer.VolumeConsolidateIndex:output_type -> volume_server_pb.VolumeConsolidateIndexResponse
+ 30, // 92: volume_server_pb.VolumeServer.VolumeDelete:output_type -> volume_server_pb.VolumeDeleteResponse
+ 32, // 93: volume_server_pb.VolumeServer.VolumeMarkReadonly:output_type -> volume_server_pb.VolumeMarkReadonlyResponse
+ 34, // 94: volume_server_pb.VolumeServer.VolumeMarkWritable:output_type -> volume_server_pb.VolumeMarkWritableResponse
+ 36, // 95: volume_server_pb.VolumeServer.VolumeConfigure:output_type -> volume_server_pb.VolumeConfigureResponse
+ 38, // 96: volume_server_pb.VolumeServer.VolumeStatus:output_type -> volume_server_pb.VolumeStatusResponse
+ 40, // 97: volume_server_pb.VolumeServer.GetState:output_type -> volume_server_pb.GetStateResponse
+ 42, // 98: volume_server_pb.VolumeServer.SetState:output_type -> volume_server_pb.SetStateResponse
+ 44, // 99: volume_server_pb.VolumeServer.VolumeCopy:output_type -> volume_server_pb.VolumeCopyResponse
+ 84, // 100: volume_server_pb.VolumeServer.ReadVolumeFileStatus:output_type -> volume_server_pb.ReadVolumeFileStatusResponse
+ 46, // 101: volume_server_pb.VolumeServer.CopyFile:output_type -> volume_server_pb.CopyFileResponse
+ 49, // 102: volume_server_pb.VolumeServer.ReceiveFile:output_type -> volume_server_pb.ReceiveFileResponse
+ 51, // 103: volume_server_pb.VolumeServer.ReadNeedleBlob:output_type -> volume_server_pb.ReadNeedleBlobResponse
+ 53, // 104: volume_server_pb.VolumeServer.ReadNeedleMeta:output_type -> volume_server_pb.ReadNeedleMetaResponse
+ 55, // 105: volume_server_pb.VolumeServer.WriteNeedleBlob:output_type -> volume_server_pb.WriteNeedleBlobResponse
+ 57, // 106: volume_server_pb.VolumeServer.ReadAllNeedles:output_type -> volume_server_pb.ReadAllNeedlesResponse
+ 59, // 107: volume_server_pb.VolumeServer.VolumeTailSender:output_type -> volume_server_pb.VolumeTailSenderResponse
+ 61, // 108: volume_server_pb.VolumeServer.VolumeTailReceiver:output_type -> volume_server_pb.VolumeTailReceiverResponse
+ 63, // 109: volume_server_pb.VolumeServer.VolumeEcShardsGenerate:output_type -> volume_server_pb.VolumeEcShardsGenerateResponse
+ 65, // 110: volume_server_pb.VolumeServer.VolumeEcShardsRebuild:output_type -> volume_server_pb.VolumeEcShardsRebuildResponse
+ 67, // 111: volume_server_pb.VolumeServer.VolumeEcShardsCopy:output_type -> volume_server_pb.VolumeEcShardsCopyResponse
+ 69, // 112: volume_server_pb.VolumeServer.VolumeEcShardsDelete:output_type -> volume_server_pb.VolumeEcShardsDeleteResponse
+ 71, // 113: volume_server_pb.VolumeServer.VolumeEcShardsMount:output_type -> volume_server_pb.VolumeEcShardsMountResponse
+ 73, // 114: volume_server_pb.VolumeServer.VolumeEcShardsUnmount:output_type -> volume_server_pb.VolumeEcShardsUnmountResponse
+ 75, // 115: volume_server_pb.VolumeServer.VolumeEcShardRead:output_type -> volume_server_pb.VolumeEcShardReadResponse
+ 77, // 116: volume_server_pb.VolumeServer.VolumeEcBlobDelete:output_type -> volume_server_pb.VolumeEcBlobDeleteResponse
+ 79, // 117: volume_server_pb.VolumeServer.VolumeEcShardsToVolume:output_type -> volume_server_pb.VolumeEcShardsToVolumeResponse
+ 81, // 118: volume_server_pb.VolumeServer.VolumeEcShardsInfo:output_type -> volume_server_pb.VolumeEcShardsInfoResponse
+ 94, // 119: volume_server_pb.VolumeServer.VolumeTierMoveDatToRemote:output_type -> volume_server_pb.VolumeTierMoveDatToRemoteResponse
+ 96, // 120: volume_server_pb.VolumeServer.VolumeTierMoveDatFromRemote:output_type -> volume_server_pb.VolumeTierMoveDatFromRemoteResponse
+ 98, // 121: volume_server_pb.VolumeServer.VolumeServerStatus:output_type -> volume_server_pb.VolumeServerStatusResponse
+ 100, // 122: volume_server_pb.VolumeServer.VolumeServerLeave:output_type -> volume_server_pb.VolumeServerLeaveResponse
+ 102, // 123: volume_server_pb.VolumeServer.FetchAndWriteNeedle:output_type -> volume_server_pb.FetchAndWriteNeedleResponse
+ 104, // 124: volume_server_pb.VolumeServer.ScrubVolume:output_type -> volume_server_pb.ScrubVolumeResponse
+ 106, // 125: volume_server_pb.VolumeServer.ScrubEcVolume:output_type -> volume_server_pb.ScrubEcVolumeResponse
+ 108, // 126: volume_server_pb.VolumeServer.Query:output_type -> volume_server_pb.QueriedStripe
+ 110, // 127: volume_server_pb.VolumeServer.VolumeNeedleStatus:output_type -> volume_server_pb.VolumeNeedleStatusResponse
+ 112, // 128: volume_server_pb.VolumeServer.Ping:output_type -> volume_server_pb.PingResponse
+ 80, // [80:129] is the sub-list for method output_type
+ 31, // [31:80] is the sub-list for method input_type
+ 31, // [31:31] is the sub-list for extension type_name
+ 31, // [31:31] is the sub-list for extension extendee
+ 0, // [0:31] is the sub-list for field type_name
}
func init() { file_volume_server_proto_init() }
diff --git a/weed/server/volume_grpc_erasure_coding.go b/weed/server/volume_grpc_erasure_coding.go
index f4a1d15ca1..14c1a4ee20 100644
--- a/weed/server/volume_grpc_erasure_coding.go
+++ b/weed/server/volume_grpc_erasure_coding.go
@@ -122,9 +122,6 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
return nil, fmt.Errorf("WriteSortedFileFromIdx %s: %v", v.IndexFileName(), err)
}
- // snapshot .dat file size before encoding — must match what .ecx references
- datSize, _, _ := v.FileStat()
-
// write .ec00 ~ .ec[TotalShards-1] files using context
ecBitrot, err := erasure_coding.WriteEcFiles(baseFileName, ecCtx)
if err != nil {
@@ -151,7 +148,10 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
}
volumeInfo := &volume_server_pb.VolumeInfo{Version: uint32(v.Version())}
volumeInfo.ExpireAtSec = expireAtSec
- volumeInfo.DatFileSize = int64(datSize)
+ // The size the encode actually read, not a separate stat: a replica-sync
+ // write can land between two stats of a live .dat, and the .vif would then
+ // record a DatFileSize and a BlockSize describing different files.
+ volumeInfo.DatFileSize = ecCtx.DatFileSize
// Validate EC configuration before saving to .vif
if ecCtx.DataShards <= 0 || ecCtx.ParityShards <= 0 || ecCtx.Total() > erasure_coding.MaxShardCount {
@@ -165,6 +165,7 @@ func (vs *VolumeServer) VolumeEcShardsGenerate(ctx context.Context, req *volume_
DataShards: uint32(ecCtx.DataShards),
ParityShards: uint32(ecCtx.ParityShards),
EncodeTsNs: time.Now().UnixNano(),
+ BlockSize: ecCtx.BlockSize,
}
glog.V(1).Infof("Saving EC config to .vif for volume %d: %d+%d (total: %d)",
req.VolumeId, ecCtx.DataShards, ecCtx.ParityShards, ecCtx.Total())
@@ -239,17 +240,22 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s
// On multi-disk servers, existing local shards may be on a different disk
// than where copied shards were placed during ec.rebuild.
rebuildDataDir := rebuildLocation.Directory
- var additionalDirs []string
- for _, otherLocation := range otherLocationsWithShards {
- additionalDirs = append(additionalDirs, otherLocation.Directory)
- }
+ additionalDirs := rebuildSearchDirs(rebuildLocation, otherLocationsWithShards)
// Rebuild missing EC files, searching all disk locations for input shards.
// Present input shards are verified against the bitrot sidecar (when present)
// and corrupt ones are regenerated; unsafe_ignore_sidecar bypasses the guard.
start := time.Now()
dataBaseFileName := path.Join(rebuildDataDir, baseFileName)
- generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName, erasure_coding.BackgroundECContext(), req.UnsafeIgnoreSidecar, additionalDirs...)
+ // Resolve the layout ONCE and use that same answer for the rebuild and for
+ // the backfill below: a manifest describing these shards has to record the
+ // geometry they were actually reconstructed with.
+ rebuildCtx, resolveErr := erasure_coding.ResolveRebuildECContext(dataBaseFileName, erasure_coding.BackgroundECContext(), additionalDirs)
+ if resolveErr != nil {
+ recordEcRebuild("failure", time.Since(start))
+ return nil, fmt.Errorf("resolve rebuild layout for %s: %v", dataBaseFileName, resolveErr)
+ }
+ generatedShardIds, err := erasure_coding.RebuildEcFiles(dataBaseFileName, rebuildCtx, req.UnsafeIgnoreSidecar, additionalDirs...)
if err != nil {
recordEcRebuild("failure", time.Since(start))
return nil, fmt.Errorf("RebuildEcFiles %s: %v", dataBaseFileName, err)
@@ -273,14 +279,19 @@ func (vs *VolumeServer) VolumeEcShardsRebuild(ctx context.Context, req *volume_s
// blesses current bytes; ComputeProtectionFromShards refuses a partial
// manifest, so a multi-server rebuild that cannot reach all shards just skips.
if erasure_coding.BitrotProtectionEnabled {
- sidecarPath := erasure_coding.BitrotSidecarPath(dataBaseFileName, 0)
- if _, statErr := os.Stat(sidecarPath); os.IsNotExist(statErr) {
- ctx := erasure_coding.NewDefaultECContext("", 0)
- if vi, _, found, _ := volume_info.MaybeLoadVolumeInfo(dataBaseFileName + ".vif"); found && vi.EcShardConfig != nil {
- if ds, ps := int(vi.EcShardConfig.DataShards), int(vi.EcShardConfig.ParityShards); ds > 0 && ps > 0 && ds+ps <= erasure_coding.MaxShardCount {
- ctx = &erasure_coding.ECContext{DataShards: ds, ParityShards: ps}
- }
- }
+ // "No sidecar yet" has to be asked of every place one could be, not
+ // just this directory. A split -dir/-dir.idx layout keeps it with the
+ // index and a sibling disk may hold it, and answering from the data
+ // base alone would write a fresh TOFU baseline over a volume that
+ // already has a manifest — blessing whatever the shards currently say
+ // and shadowing the real record, since the data base is searched first.
+ if erasure_coding.FindBitrotSidecar(0, dataBaseFileName, indexBaseFileName, additionalDirs...) == "" {
+ sidecarPath := erasure_coding.BitrotSidecarPath(dataBaseFileName, 0)
+ // The manifest must describe the shards as rebuilt, so it takes the
+ // context the rebuild resolved — not a narrower re-derivation that
+ // reads only this directory's .vif and drops the block size, which
+ // records the legacy layout for shards written with a uniform one.
+ ctx := rebuildCtx
if prot, berr := erasure_coding.ComputeProtectionFromShards(dataBaseFileName, ctx, 0, additionalDirs); berr != nil {
glog.V(2).Infof("bitrot backfill skipped for %s: %v", dataBaseFileName, berr)
} else if werr := erasure_coding.SaveBitrotSidecar(sidecarPath, prot); werr != nil {
@@ -710,6 +721,36 @@ func removeBitrotSidecars(baseFilename string) error {
return firstErr
}
+// rebuildSearchDirs lists every directory besides the rebuild's own data
+// directory that a rebuild may have to read from. Shards are only half of it:
+// a split -dir/-dir.idx layout keeps .ecx/.ecj/.vif with the index, and on a
+// multi-disk server the chosen disk may hold nothing but shards while the
+// volume's .vif or generation-0 .ecsum sits on a sibling. Miss those and the
+// layout resolution falls back to the default ratio and the legacy block
+// size, reconstructing through the wrong matrix. The rebuild's own INDEX
+// directory belongs here too — callers pass the data-directory base name, so
+// it is not otherwise searched. Empty and duplicate entries are dropped.
+func rebuildSearchDirs(rebuildLocation *storage.DiskLocation, otherLocations []*storage.DiskLocation) []string {
+ var dirs []string
+ appendDir := func(dir string) {
+ if dir == "" || dir == rebuildLocation.Directory {
+ return
+ }
+ for _, existing := range dirs {
+ if existing == dir {
+ return
+ }
+ }
+ dirs = append(dirs, dir)
+ }
+ appendDir(rebuildLocation.IdxDirectory)
+ for _, otherLocation := range otherLocations {
+ appendDir(otherLocation.Directory)
+ appendDir(otherLocation.IdxDirectory)
+ }
+ return dirs
+}
+
func checkEcVolumeStatus(bName string, location *storage.DiskLocation) (hasEcxFile bool, hasIdxFile bool, existingShardCount int, err error) {
// check whether to delete the .ecx and .ecj file also
fileInfos, err := os.ReadDir(location.Directory)
@@ -782,6 +823,27 @@ func (vs *VolumeServer) VolumeEcShardsMount(ctx context.Context, req *volume_ser
}
}
+ // A shard delivery can bring the checksum manifest with it, but the receive
+ // path only writes the file. When this server already had the volume
+ // mounted, the EcVolume in memory keeps whatever protection state it
+ // resolved at mount — off, for a volume whose sidecar arrives now — until a
+ // remount. Re-resolve it here, where the shards it describes were added.
+ // Every per-disk runtime, not just the first: a vid mounts as one EcVolume
+ // per disk, the delivery lands the .ecsum on one of them, and the
+ // first-match FindEcVolume would leave the siblings reporting no protection
+ // until a remount. Each re-resolves against its own data and index base, so
+ // a shared -dir.idx reaches all of them.
+ //
+ // Resolving across every EC metadata directory is what makes that reload
+ // mean something. Startup mirroring gives each shard-bearing disk its own
+ // .ecx/.ecj/.vif but deliberately not the sidecar, so a runtime restricted
+ // to its own two directories would find nothing however often it reloaded.
+ // One delivered copy, reachable from all of them.
+ ecMetadataDirs := vs.store.EcMetadataDirs()
+ for _, v := range vs.store.FindAllEcVolumes(needle.VolumeId(req.VolumeId)) {
+ v.ReloadBitrotSidecar(ecMetadataDirs...)
+ }
+
return &volume_server_pb.VolumeEcShardsMountResponse{}, nil
}
@@ -1003,7 +1065,7 @@ func (vs *VolumeServer) VolumeEcShardsToVolume(ctx context.Context, req *volume_
// boundary, so the layout must not be derived from datFileSize. WriteDatFile
// infers the layout from the shard size when .vif does not record it.
// write .dat file from .ec00 ~ .ec09 files
- if err := erasure_coding.WriteDatFile(dataBaseFileName, datFileSize, v.DatFileSize(), shardFileNames); err != nil {
+ if err := erasure_coding.WriteDatFile(dataBaseFileName, datFileSize, v.DatFileSize(), shardFileNames, v.ECContext.LargeBlockSize(), v.ECContext.SmallBlockSize()); err != nil {
return nil, fmt.Errorf("WriteDatFile %s: %v", dataBaseFileName, err)
}
@@ -1179,11 +1241,25 @@ func (vs *VolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_serv
return nil, err
}
+ // Report the layout this holder will actually serve reads through. It is
+ // the only way a coordinator can tell a holder that understands the
+ // uniform block layout from one that dropped the unknown .vif field on the
+ // floor and mounted the volume as legacy.
+ var ecShardConfig *volume_server_pb.EcShardConfig
+ if primary.ECContext != nil {
+ ecShardConfig = &volume_server_pb.EcShardConfig{
+ DataShards: uint32(primary.ECContext.DataShards),
+ ParityShards: uint32(primary.ECContext.ParityShards),
+ BlockSize: primary.ECContext.BlockSize,
+ }
+ }
+
res := &volume_server_pb.VolumeEcShardsInfoResponse{
EcShardInfos: shardInfos,
FileCount: files,
FileDeletedCount: filesDeleted,
VolumeSize: totalSize,
+ EcShardConfig: ecShardConfig,
}
return res, nil
diff --git a/weed/server/volume_grpc_erasure_coding_dirs_test.go b/weed/server/volume_grpc_erasure_coding_dirs_test.go
new file mode 100644
index 0000000000..4516fca6fa
--- /dev/null
+++ b/weed/server/volume_grpc_erasure_coding_dirs_test.go
@@ -0,0 +1,73 @@
+package weed_server
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/storage"
+)
+
+func loc(dir, idxDir string) *storage.DiskLocation {
+ return &storage.DiskLocation{Directory: dir, IdxDirectory: idxDir}
+}
+
+// The rebuild reads its shards from one directory but resolves the volume's
+// layout — ratio and uniform block size — from the .vif or the generation-0
+// .ecsum, which on a multi-disk server may sit anywhere. Every directory that
+// could hold one has to be in the search list, or the resolution silently
+// falls back to 10+4 with the legacy striping and reconstructs through the
+// wrong matrix.
+func TestRebuildSearchDirs(t *testing.T) {
+ tests := []struct {
+ name string
+ rebuild *storage.DiskLocation
+ others []*storage.DiskLocation
+ want []string
+ }{
+ {
+ name: "the rebuild's own index directory is searched",
+ rebuild: loc("/data1", "/idx1"),
+ want: []string{"/idx1"},
+ },
+ {
+ name: "an unsplit rebuild location contributes nothing",
+ rebuild: loc("/data1", "/data1"),
+ want: nil,
+ },
+ {
+ // The case two reviewers flagged: a sibling holding only shards
+ // while its index directory holds this volume's .vif.
+ name: "a sibling's index directory is searched, not just its data directory",
+ rebuild: loc("/data1", "/data1"),
+ others: []*storage.DiskLocation{loc("/data2", "/idx2")},
+ want: []string{"/data2", "/idx2"},
+ },
+ {
+ name: "shared index directories are listed once",
+ rebuild: loc("/data1", "/shared-idx"),
+ others: []*storage.DiskLocation{loc("/data2", "/shared-idx"), loc("/data3", "/shared-idx")},
+ want: []string{"/shared-idx", "/data2", "/data3"},
+ },
+ {
+ name: "the rebuild's data directory is never repeated",
+ rebuild: loc("/data1", "/idx1"),
+ others: []*storage.DiskLocation{loc("/data1", "/idx1"), loc("/data2", "/data1")},
+ want: []string{"/idx1", "/data2"},
+ },
+ {
+ name: "empty directories are dropped",
+ rebuild: loc("/data1", ""),
+ others: []*storage.DiskLocation{loc("/data2", "")},
+ want: []string{"/data2"},
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ got := rebuildSearchDirs(tc.rebuild, tc.others)
+ if !reflect.DeepEqual(got, tc.want) {
+ t.Errorf("rebuildSearchDirs() = %v, want %v", got, tc.want)
+ }
+ })
+ }
+}
diff --git a/weed/server/volume_grpc_erasure_coding_recover_test.go b/weed/server/volume_grpc_erasure_coding_recover_test.go
index 9f21edc94e..fc0d0d1ab4 100644
--- a/weed/server/volume_grpc_erasure_coding_recover_test.go
+++ b/weed/server/volume_grpc_erasure_coding_recover_test.go
@@ -18,6 +18,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
+ "github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
"github.com/seaweedfs/seaweedfs/weed/util"
)
@@ -111,7 +112,12 @@ func TestFetchEcIndexFromPeers_CopiesIndexOverGrpc(t *testing.T) {
if err := os.WriteFile(srcBase+".ecj", []byte("journal"), 0o644); err != nil {
t.Fatalf("write source .ecj: %v", err)
}
- if err := os.WriteFile(srcBase+".vif", []byte("volinfo"), 0o644); err != nil {
+ // A real .vif: the receiver MOUNTS the volume from the copied files, and a
+ // mount refuses a .vif it cannot parse rather than guessing the layout.
+ if err := volume_info.SaveVolumeInfo(srcBase+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4},
+ }); err != nil {
t.Fatalf("write source .vif: %v", err)
}
@@ -223,7 +229,12 @@ func TestVolumeEcShardsMount_RecoverMissingIndex(t *testing.T) {
if err := os.WriteFile(srcBase+".ecj", nil, 0o644); err != nil {
t.Fatalf("write source .ecj: %v", err)
}
- if err := os.WriteFile(srcBase+".vif", []byte("volinfo"), 0o644); err != nil {
+ // A real .vif: the receiver MOUNTS the volume from the copied files, and a
+ // mount refuses a .vif it cannot parse rather than guessing the layout.
+ if err := volume_info.SaveVolumeInfo(srcBase+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4},
+ }); err != nil {
t.Fatalf("write source .vif: %v", err)
}
srcGrpcPort := serveGrpc(t, func(s *grpc.Server) {
diff --git a/weed/storage/disk_location_ec.go b/weed/storage/disk_location_ec.go
index 218abc8d60..71987b550e 100644
--- a/weed/storage/disk_location_ec.go
+++ b/weed/storage/disk_location_ec.go
@@ -484,26 +484,17 @@ func (l *DiskLocation) checkOrphanedShards(shards []string, collection string, v
// erasure_coding.DataShardsCount so that tests writing a custom layout
// to .vif compute the matching shard size, and so custom-ratio builds
// (e.g. enterprise) can swap the default without touching this helper.
+// The padded shard length is the same number under both layouts —
+// TestUniformBlockSizeMatchesLegacyShardSize asserts the equivalence for every
+// input — so defer to the encoder's own helper instead of keeping a second copy
+// of the padding rule that a future change would have to be made in twice. An
+// empty .dat keeps its historic answer: the legacy encoder emits no block for
+// it, where UniformBlockSize floors at one.
func calculateExpectedShardSize(datFileSize int64, dataShardCount int) int64 {
- if dataShardCount <= 0 {
+ if dataShardCount <= 0 || datFileSize <= 0 {
return 0
}
- var shardSize int64
-
- // Process large blocks (1GB * dataShardCount per batch)
- largeBatchSize := int64(erasure_coding.ErasureCodingLargeBlockSize) * int64(dataShardCount)
- numLargeBatches := datFileSize / largeBatchSize
- shardSize = numLargeBatches * int64(erasure_coding.ErasureCodingLargeBlockSize)
- remainingSize := datFileSize - (numLargeBatches * largeBatchSize)
-
- // Process remaining data in small blocks (1MB * dataShardCount per batch)
- if remainingSize > 0 {
- smallBatchSize := int64(erasure_coding.ErasureCodingSmallBlockSize) * int64(dataShardCount)
- numSmallBatches := (remainingSize + smallBatchSize - 1) / smallBatchSize // Ceiling division
- shardSize += numSmallBatches * int64(erasure_coding.ErasureCodingSmallBlockSize)
- }
-
- return shardSize
+ return erasure_coding.UniformBlockSize(datFileSize, dataShardCount)
}
// validateEcVolume reports whether the EC files for (collection, vid) on this
diff --git a/weed/storage/erasure_coding/ec_bitrot.go b/weed/storage/erasure_coding/ec_bitrot.go
index 5aef042c75..dfe78a5f2c 100644
--- a/weed/storage/erasure_coding/ec_bitrot.go
+++ b/weed/storage/erasure_coding/ec_bitrot.go
@@ -195,6 +195,7 @@ func buildProtectionFromBuilders(ctx *ECContext, builders []*shardChecksumBuilde
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: uint32(ctx.DataShards),
ParityShards: uint32(ctx.ParityShards),
+ BlockSize: ctx.BlockSize,
},
Shards: shards,
EncodeUuid: NewEncodeUUID(),
@@ -431,6 +432,7 @@ func ComputeProtectionFromShards(baseFileName string, ctx *ECContext, generation
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: uint32(ctx.DataShards),
ParityShards: uint32(ctx.ParityShards),
+ BlockSize: ctx.BlockSize,
},
Shards: shards,
EncodeUuid: NewEncodeUUID(),
@@ -474,53 +476,141 @@ func (ev *EcVolume) BitrotProtection() (*volume_server_pb.EcBitrotProtection, Bi
return ev.bitrot, ev.bitrotStatus
}
+// ReloadBitrotSidecar re-resolves the checksum sidecar for a volume that is
+// already mounted — a shard delivery can bring the manifest with it, and the
+// receive path only writes the file, so without this the in-memory volume
+// keeps the protection state it resolved at mount (off) until a remount.
+func (ev *EcVolume) ReloadBitrotSidecar(additionalDirs ...string) {
+ if err := ev.loadActiveBitrotSidecar(additionalDirs...); err != nil {
+ glog.Warningf("ec volume %d: reload bitrot sidecar: %v", ev.VolumeId, err)
+ }
+}
+
// loadActiveBitrotSidecar loads the generation-0 checksum sidecar into the
// volume. Best-effort: any failure leaves protection off/invalid without
// failing the mount. OSS only produces generation-0 (fresh-encode) sidecars.
-func (ev *EcVolume) loadActiveBitrotSidecar() {
- ev.loadBitrotForGeneration(0)
+func (ev *EcVolume) loadActiveBitrotSidecar(additionalDirs ...string) error {
+ return ev.loadBitrotForGeneration(0, additionalDirs...)
}
// loadBitrotForGeneration loads and validates the sidecar describing generation
// `generation`, setting ev.bitrot/ev.bitrotStatus. Called at mount. Absent or
-// generation/config-mismatched => BitrotOff; self-integrity/manifest failure =>
+// generation-mismatched => BitrotOff; self-integrity/manifest failure =>
// BitrotInvalid; usable => BitrotOn.
-func (ev *EcVolume) loadBitrotForGeneration(generation uint32) {
+//
+// A sidecar written FOR THIS generation that disagrees with the volume's shard
+// geometry is different in kind from those: both files record the layout the
+// generation was encoded with, so a disagreement means one of them is wrong and
+// reads through the other would land at the wrong shard offsets. That returns
+// an error and fails the mount instead of quietly dropping to unprotected
+// reads.
+func (ev *EcVolume) loadBitrotForGeneration(generation uint32, additionalDirs ...string) error {
ev.bitrotLock.Lock()
defer ev.bitrotLock.Unlock()
ev.bitrot = nil
ev.bitrotStatus = BitrotOff
if ev.ECContext == nil {
- return
+ return nil
}
- path := findBitrotSidecar(generation, ev.DataBaseFileName(), ev.IndexBaseFileName())
+ path := findBitrotSidecar(generation, ev.DataBaseFileName(), ev.IndexBaseFileName(), additionalDirs...)
if path == "" {
- return
+ return nil
}
prot, err := LoadBitrotSidecar(path)
if err != nil {
glog.Warningf("ec volume %d: bitrot sidecar %s self-integrity failed: %v", ev.VolumeId, path, err)
ev.bitrotStatus = BitrotInvalid
- return
+ return nil
}
if prot.Generation != generation {
- return // not for this generation -> off, not corruption
+ return nil // not for this generation -> off, not corruption
+ }
+ if prot.EcShardConfig == nil {
+ return nil // records no geometry -> nothing to contradict
}
- if prot.EcShardConfig == nil ||
- int(prot.EcShardConfig.DataShards) != ev.ECContext.DataShards ||
- int(prot.EcShardConfig.ParityShards) != ev.ECContext.ParityShards {
- return
+ if int(prot.EcShardConfig.DataShards) != ev.ECContext.DataShards ||
+ int(prot.EcShardConfig.ParityShards) != ev.ECContext.ParityShards ||
+ prot.EcShardConfig.BlockSize != ev.ECContext.BlockSize {
+ return fmt.Errorf("ec volume %d generation %d: %s records layout %d+%d block %d but the volume is mounted as %d+%d block %d; refusing to serve one of the two layouts",
+ ev.VolumeId, generation, path,
+ prot.EcShardConfig.DataShards, prot.EcShardConfig.ParityShards, prot.EcShardConfig.BlockSize,
+ ev.ECContext.DataShards, ev.ECContext.ParityShards, ev.ECContext.BlockSize)
}
if err := ValidateBitrotManifest(prot, ev.ECContext.DataShards, ev.ECContext.ParityShards); err != nil {
glog.Warningf("ec volume %d: bitrot sidecar %s manifest invalid: %v", ev.VolumeId, path, err)
ev.bitrotStatus = BitrotInvalid
- return
+ return nil
}
ev.bitrot = prot
ev.bitrotStatus = BitrotOn
glog.V(1).Infof("ec volume %d: loaded bitrot protection generation %d (%d shards, block_size %d)",
ev.VolumeId, generation, len(prot.Shards), prot.BlockSize)
+ return nil
+}
+
+// layoutFromSidecar resolves a volume's EC layout from its generation-0
+// checksum sidecar, searching the data base and then the index base. A split
+// -dir/-dir.idx layout keeps the sidecar with the index, so probing the data
+// base alone reports "absent" for a volume that has the record right there —
+// and absent is the one answer that selects the legacy 10+4 layout.
+//
+// The three answers stay distinct: (nil, false, nil) means genuinely absent
+// and the caller may fall back to the defaults; a non-nil error means the
+// record exists but cannot establish the layout, which must fail rather than
+// default; otherwise the config is usable.
+func layoutFromSidecar(dataBaseFileName, indexBaseFileName string) (cfg *volume_server_pb.EcShardConfig, found bool, err error) {
+ path := findBitrotSidecar(0, dataBaseFileName, indexBaseFileName)
+ if path == "" {
+ return nil, false, nil
+ }
+ cfg, err = EcShardConfigFromSidecarPath(path)
+ return cfg, true, err
+}
+
+// EcShardConfigFromSidecar reads the generation-0 bitrot sidecar's record of a
+// volume's EC config. The three answers are distinct on purpose: absent means
+// the volume may genuinely predate the sidecar, which is the only case a caller
+// may answer with the legacy layout; present-but-unusable means the one
+// surviving record of the geometry is corrupt, and guessing from there returns
+// wrong bytes rather than no bytes.
+func EcShardConfigFromSidecar(baseFileName string) (cfg *volume_server_pb.EcShardConfig, found bool, err error) {
+ path := BitrotSidecarPath(baseFileName, 0)
+ if _, statErr := os.Stat(path); statErr != nil {
+ if os.IsNotExist(statErr) {
+ return nil, false, nil
+ }
+ return nil, false, fmt.Errorf("stat %s: %w", path, statErr)
+ }
+ cfg, err = EcShardConfigFromSidecarPath(path)
+ return cfg, true, err
+}
+
+// EcShardConfigFromSidecarPath is EcShardConfigFromSidecar for a sidecar whose
+// path the caller already resolved — the rebuild finds it across a multi-disk
+// server's directories, not only next to the base name.
+func EcShardConfigFromSidecarPath(path string) (*volume_server_pb.EcShardConfig, error) {
+ prot, loadErr := LoadBitrotSidecar(path)
+ if loadErr != nil {
+ return nil, fmt.Errorf("read %s: %w", path, loadErr)
+ }
+ // The un-suffixed sidecar describes generation 0 and nothing else; one
+ // stamped for another generation is not a record of these shards.
+ if prot.GetGeneration() != 0 {
+ return nil, fmt.Errorf("%s records generation %d, not generation 0", path, prot.GetGeneration())
+ }
+ cfg := prot.GetEcShardConfig()
+ if cfg == nil {
+ return nil, fmt.Errorf("%s records no EC config", path)
+ }
+ if !ValidEcShardCounts(cfg.GetDataShards(), cfg.GetParityShards()) {
+ return nil, fmt.Errorf("%s records invalid shard counts %d+%d",
+ path, cfg.GetDataShards(), cfg.GetParityShards())
+ }
+ if bsErr := ValidateBlockSize(cfg.GetBlockSize()); bsErr != nil {
+ return nil, fmt.Errorf("%s: %w", path, bsErr)
+ }
+ return cfg, nil
}
// RemoveBitrotSidecars removes the legacy <base>.ecsum and any versioned
@@ -534,6 +624,15 @@ func RemoveBitrotSidecars(base string) {
}
}
+// FindBitrotSidecar is findBitrotSidecar for callers outside this package. It
+// answers "does this volume already have a manifest, and where" — a question
+// that cannot be settled from one base name, because a split -dir/-dir.idx
+// layout keeps the sidecar with the index and a multi-disk server may keep it
+// on a sibling. Returns "" when no candidate exists.
+func FindBitrotSidecar(generation uint32, dataBase, indexBase string, additionalDirs ...string) string {
+ return findBitrotSidecar(generation, dataBase, indexBase, additionalDirs...)
+}
+
// findBitrotSidecar resolves the sidecar path for a generation, searching the
// data base, the index base, and any additional directories — mirroring how
// shard/.vif lookups handle split data/idx layouts and per-disk mirrors.
diff --git a/weed/storage/erasure_coding/ec_context.go b/weed/storage/erasure_coding/ec_context.go
index 9fc352d83b..ac77f688b5 100644
--- a/weed/storage/erasure_coding/ec_context.go
+++ b/weed/storage/erasure_coding/ec_context.go
@@ -13,6 +13,16 @@ type ECContext struct {
ParityShards int
Collection string
VolumeId needle.VolumeId
+ // BlockSize > 0 selects the uniform block layout: every shard is a single
+ // contiguous block of this many bytes, so consecutive .dat ranges stay on
+ // one shard instead of striping across all of them at 1MiB granularity.
+ // 0 is the legacy two-tier 1GiB/1MiB layout.
+ BlockSize int64
+ // DatFileSize is the length of the .dat the encode actually read, set by
+ // WriteEcFiles from the same measurement it derives BlockSize from. The
+ // .vif must record the two together: they describe one file, and a live
+ // volume can grow between two separate stats.
+ DatFileSize int64
}
// Total returns the total number of shards (data + parity)
@@ -20,6 +30,33 @@ func (ctx *ECContext) Total() int {
return ctx.DataShards + ctx.ParityShards
}
+// ValidEcShardCounts reports whether a recorded (data, parity) pair could
+// describe a real EC volume. The sum is taken in uint64 deliberately: on a
+// 32-bit build `int` is 32 bits, so converting each count first and adding
+// them wraps for values near the uint32 ceiling — 0x7fffffff + 0x7fffffff
+// lands at -2, which slips under the MaxShardCount bound.
+func ValidEcShardCounts(dataShards, parityShards uint32) bool {
+ return dataShards > 0 && parityShards > 0 &&
+ uint64(dataShards)+uint64(parityShards) <= uint64(MaxShardCount)
+}
+
+// LargeBlockSize returns the large-block length of this context's shard
+// layout; nil-safe so callers can pass an unset context for the legacy layout.
+func (ctx *ECContext) LargeBlockSize() int64 {
+ if ctx != nil && ctx.BlockSize > 0 {
+ return ctx.BlockSize
+ }
+ return ErasureCodingLargeBlockSize
+}
+
+// SmallBlockSize returns the small-block length of this context's shard layout.
+func (ctx *ECContext) SmallBlockSize() int64 {
+ if ctx != nil && ctx.BlockSize > 0 {
+ return ctx.BlockSize
+ }
+ return ErasureCodingSmallBlockSize
+}
+
// NewDefaultECContext creates a context with default 10+4 shard configuration
func NewDefaultECContext(collection string, volumeId needle.VolumeId) *ECContext {
return &ECContext{
diff --git a/weed/storage/erasure_coding/ec_decoder.go b/weed/storage/erasure_coding/ec_decoder.go
index 5bf4342d30..96494c4a5a 100644
--- a/weed/storage/erasure_coding/ec_decoder.go
+++ b/weed/storage/erasure_coding/ec_decoder.go
@@ -233,8 +233,10 @@ func iterateEcjFile(baseFileName string, processNeedleFn func(key types.NeedleId
// large-block row boundary, and deriving the layout from the shrunk extent
// would read the shards in the wrong block order. Pass zero when the .vif does
// not record the encode-time size to infer the layout from the shard size.
-func WriteDatFile(baseFileName string, datFileSize int64, encodedDatFileSize int64, shardFileNames []string) error {
- return writeDatFile(baseFileName, datFileSize, encodedDatFileSize, shardFileNames, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize)
+// largeBlockSize/smallBlockSize are the volume's shard block layout, e.g.
+// ctx.LargeBlockSize()/ctx.SmallBlockSize() from its .vif EC config.
+func WriteDatFile(baseFileName string, datFileSize int64, encodedDatFileSize int64, shardFileNames []string, largeBlockSize int64, smallBlockSize int64) error {
+ return writeDatFile(baseFileName, datFileSize, encodedDatFileSize, shardFileNames, largeBlockSize, smallBlockSize)
}
func writeDatFile(baseFileName string, datFileSize int64, encodedDatFileSize int64, shardFileNames []string, largeBlockSize int64, smallBlockSize int64) error {
@@ -288,7 +290,11 @@ func writeDatFile(baseFileName string, datFileSize int64, encodedDatFileSize int
// A shard size that is an exact multiple of the large block size is
// ambiguous: N large rows, or N-1 large rows plus a full small-block
// region. The two layouts only agree below the last large row.
- if shardSize%largeBlockSize == 0 && datFileSize > (shardSize/largeBlockSize-1)*largeBlockSize*int64(dataShards) {
+ // A uniform layout has no such ambiguity — its large and small blocks
+ // are the same size, so every reading of the shard is the same one, and
+ // without this precondition the check fires on every volume.
+ if largeBlockSize != smallBlockSize &&
+ shardSize%largeBlockSize == 0 && datFileSize > (shardSize/largeBlockSize-1)*largeBlockSize*int64(dataShards) {
return fmt.Errorf("shard size %d of %s does not identify the block layout; re-encode to record the dat size in .vif", shardSize, baseFileName)
}
encodedDatFileSize = int64(dataShards) * shardSize
diff --git a/weed/storage/erasure_coding/ec_decoder_test.go b/weed/storage/erasure_coding/ec_decoder_test.go
index 8c655d7fa1..b09fed9479 100644
--- a/weed/storage/erasure_coding/ec_decoder_test.go
+++ b/weed/storage/erasure_coding/ec_decoder_test.go
@@ -217,7 +217,7 @@ func TestDecodeAtomicPublish(t *testing.T) {
// final .dat nor a partial .dat.tmp behind.
datBase := filepath.Join(dir, "bar_2")
missingShards := []string{filepath.Join(dir, "does_not_exist.ec00")}
- if err := erasure_coding.WriteDatFile(datBase, 100, 100, missingShards); err == nil {
+ if err := erasure_coding.WriteDatFile(datBase, 100, 100, missingShards, erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize); err == nil {
t.Fatalf("expected WriteDatFile to fail on missing shard")
}
if _, err := os.Stat(datBase + ".dat"); !os.IsNotExist(err) {
diff --git a/weed/storage/erasure_coding/ec_encoder.go b/weed/storage/erasure_coding/ec_encoder.go
index 927a50d865..1b28fddf16 100644
--- a/weed/storage/erasure_coding/ec_encoder.go
+++ b/weed/storage/erasure_coding/ec_encoder.go
@@ -62,26 +62,76 @@ func WriteSortedFileFromIdx(baseFileName string, ext string) (e error) {
// BackgroundECContext for the default ratio, or an explicit ctx for a configured
// (e.g. custom-ratio) layout. It returns the bitrot protection (per-shard block
// CRC32C) computed during the single encode pass; the caller persists it as a
-// <base>.ecsum sidecar.
+// <base>.ecsum sidecar, and persists ctx.BlockSize and ctx.DatFileSize (both
+// set here, from one measurement of the .dat) to the .vif so readers resolve
+// the shard block layout.
func WriteEcFiles(baseFileName string, ctx *ECContext) (*volume_server_pb.EcBitrotProtection, error) {
- if ctx == nil || ctx.Total() == 0 {
+ if ctx == nil {
ctx = NewDefaultECContext("", 0)
+ } else if ctx.Total() == 0 {
+ // Fill the placeholder in place rather than swapping the pointer: the
+ // caller reads BlockSize and DatFileSize back off the context it
+ // passed, and a replacement leaves it holding the zero values.
+ ctx.DataShards, ctx.ParityShards = DataShardsCount, ParityShardsCount
+ }
+ // Always encode with the uniform block layout, sized for this .dat. Both
+ // the block size and the .dat length it was derived from are left on ctx,
+ // so the caller persists a .vif whose two fields describe one measurement
+ // — a second stat could see a different size on a volume still taking
+ // writes.
+ fi, err := os.Stat(baseFileName + ".dat")
+ if err != nil {
+ return nil, fmt.Errorf("failed to stat dat file: %w", err)
}
- return generateEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx)
+ ctx.DatFileSize = fi.Size()
+ ctx.BlockSize = UniformBlockSize(fi.Size(), ctx.DataShards)
+ return generateEcFiles(baseFileName, 256*1024, ctx.BlockSize, ctx.BlockSize, ctx)
}
-// RebuildEcFiles rebuilds missing EC shard files. Pass BackgroundECContext to
-// resolve the layout from the volume's .vif (falling back to the default ratio),
-// or an explicit ctx when the caller already knows the shard layout.
-// additionalDirs are extra directories to search for existing shard files,
-// which handles multi-disk servers where shards may be spread across disks.
-// When a bitrot checksum sidecar is present for the (generation-0) volume,
-// present input shards are verified against it and corrupt ones are excluded
-// from Reed-Solomon and regenerated; unsafeIgnoreSidecar bypasses that guard.
-func RebuildEcFiles(baseFileName string, ctx *ECContext, unsafeIgnoreSidecar bool, additionalDirs ...string) ([]uint32, error) {
+// ValidateBlockSize reports whether a `.vif`-recorded shard block size is one
+// an encoder could have produced. 0 means the legacy two-tier layout, which is
+// always valid; anything positive must be a whole number of small blocks,
+// because that is what UniformBlockSize rounds to. A negative or unaligned
+// value is corruption, and using it would map every read to the wrong shard
+// offset.
+func ValidateBlockSize(blockSize int64) error {
+ if blockSize == 0 {
+ return nil
+ }
+ if blockSize < 0 || blockSize%ErasureCodingSmallBlockSize != 0 {
+ return fmt.Errorf("invalid shard block size %d: expected 0 (legacy) or a multiple of %d",
+ blockSize, ErasureCodingSmallBlockSize)
+ }
+ return nil
+}
+
+// UniformBlockSize returns the per-shard block size of the uniform layout for
+// a .dat of the given size: ceil(datFileSize/dataShards) rounded up to a whole
+// small block. For every input this equals the legacy layout's padded shard
+// size, so only the byte placement differs between the two layouts, never the
+// shard length.
+func UniformBlockSize(datFileSize int64, dataShards int) int64 {
+ perShard := (datFileSize + int64(dataShards) - 1) / int64(dataShards)
+ blocks := (perShard + ErasureCodingSmallBlockSize - 1) / ErasureCodingSmallBlockSize
+ if blocks < 1 {
+ blocks = 1
+ }
+ return blocks * ErasureCodingSmallBlockSize
+}
+
+// ResolveRebuildECContext answers which shard layout a rebuild of baseFileName
+// will use: the caller's context when it already states one, else the volume's
+// own metadata — its `.vif` in any of the directories the rebuild can read,
+// then the generation-0 bitrot sidecar, and only then the build defaults.
+// Exported so a caller that must agree with the rebuild (the post-rebuild
+// bitrot backfill writes a manifest describing these very shards) resolves
+// once and uses the same answer, instead of re-deriving it from a narrower
+// search and recording a layout the rebuild did not use.
+func ResolveRebuildECContext(baseFileName string, ctx *ECContext, additionalDirs []string) (*ECContext, error) {
if ctx == nil || ctx.Total() == 0 {
// Resolve the layout from the .vif to preserve the original configuration.
- volumeInfo, _, foundVif, vifErr := volume_info.MaybeLoadVolumeInfo(baseFileName + ".vif")
+ vifPath := findVifPath(baseFileName, additionalDirs)
+ volumeInfo, _, foundVif, vifErr := volume_info.MaybeLoadVolumeInfo(vifPath)
if vifErr != nil {
// The .vif exists but cannot be read or parsed. Fail closed rather
// than silently falling back to the default ratio, which would
@@ -89,28 +139,65 @@ func RebuildEcFiles(baseFileName string, ctx *ECContext, unsafeIgnoreSidecar boo
// explicit ctx to override.
return nil, fmt.Errorf("RebuildEcFiles %s: cannot load .vif: %w", baseFileName, vifErr)
}
- if foundVif && volumeInfo.EcShardConfig != nil {
- ds := int(volumeInfo.EcShardConfig.DataShards)
- ps := int(volumeInfo.EcShardConfig.ParityShards)
-
- // Validate EC config before using it
- if ds > 0 && ps > 0 && ds+ps <= MaxShardCount {
+ switch {
+ case foundVif && volumeInfo.EcShardConfig != nil &&
+ ValidEcShardCounts(volumeInfo.EcShardConfig.DataShards, volumeInfo.EcShardConfig.ParityShards):
+ if bsErr := ValidateBlockSize(volumeInfo.EcShardConfig.GetBlockSize()); bsErr != nil {
+ return nil, fmt.Errorf("RebuildEcFiles %s: %s: %w", baseFileName, vifPath, bsErr)
+ }
+ ctx = &ECContext{
+ DataShards: int(volumeInfo.EcShardConfig.DataShards),
+ ParityShards: int(volumeInfo.EcShardConfig.ParityShards),
+ BlockSize: volumeInfo.EcShardConfig.GetBlockSize(),
+ }
+ glog.V(0).Infof("Rebuilding EC files for %s with config from .vif: %s", baseFileName, ctx.String())
+ case foundVif && volumeInfo.EcShardConfig != nil:
+ // A recorded-but-impossible ratio is corruption, not a reason to
+ // substitute the default one: a 12+4 volume rebuilt as 10+4
+ // reconstructs from the wrong matrix and never regenerates shards
+ // 14-15. Pass an explicit ctx to override.
+ return nil, fmt.Errorf("RebuildEcFiles %s: %s records invalid shard counts %d+%d",
+ baseFileName, vifPath, volumeInfo.EcShardConfig.DataShards, volumeInfo.EcShardConfig.ParityShards)
+ default:
+ // No usable .vif: the bitrot sidecar records the same config at
+ // encode time and is then the surviving authority. Reading the
+ // default ratio and the legacy block size instead would rebuild
+ // from the wrong geometry AND make the sidecar look like it
+ // disagrees, which silently skips every checksum check below.
+ if sidecarPath := findBitrotSidecar(0, baseFileName, baseFileName, additionalDirs...); sidecarPath != "" {
+ cfg, cfgErr := EcShardConfigFromSidecarPath(sidecarPath)
+ if cfgErr != nil {
+ return nil, fmt.Errorf("RebuildEcFiles %s: no usable .vif and %w", baseFileName, cfgErr)
+ }
ctx = &ECContext{
- DataShards: ds,
- ParityShards: ps,
+ DataShards: int(cfg.GetDataShards()),
+ ParityShards: int(cfg.GetParityShards()),
+ BlockSize: cfg.GetBlockSize(),
}
- glog.V(0).Infof("Rebuilding EC files for %s with config from .vif: %s", baseFileName, ctx.String())
- } else {
- glog.Warningf("Invalid EC config in .vif for %s (data=%d, parity=%d), using default", baseFileName, ds, ps)
- ctx = NewDefaultECContext("", 0)
+ glog.V(0).Infof("Rebuilding EC files for %s with config from the bitrot sidecar: %s", baseFileName, ctx.String())
+ break
}
- } else {
glog.V(0).Infof("Rebuilding EC files for %s with default config", baseFileName)
ctx = NewDefaultECContext("", 0)
}
}
+ return ctx, nil
+}
- return generateMissingEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx, unsafeIgnoreSidecar, additionalDirs)
+// RebuildEcFiles rebuilds missing EC shard files. Pass BackgroundECContext to
+// resolve the layout from the volume's .vif (falling back to the default ratio),
+// or an explicit ctx when the caller already knows the shard layout.
+// additionalDirs are extra directories to search for existing shard files,
+// which handles multi-disk servers where shards may be spread across disks.
+// When a bitrot checksum sidecar is present for the (generation-0) volume,
+// present input shards are verified against it and corrupt ones are excluded
+// from Reed-Solomon and regenerated; unsafeIgnoreSidecar bypasses that guard.
+func RebuildEcFiles(baseFileName string, ctx *ECContext, unsafeIgnoreSidecar bool, additionalDirs ...string) ([]uint32, error) {
+ ctx, err := ResolveRebuildECContext(baseFileName, ctx, additionalDirs)
+ if err != nil {
+ return nil, err
+ }
+ return generateMissingEcFiles(baseFileName, 256*1024, ctx, unsafeIgnoreSidecar, additionalDirs)
}
func ToExt(ecIndex int) string {
@@ -159,7 +246,10 @@ func findShardFile(baseFileName string, ext string, additionalDirs []string) str
return ""
}
-func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext, unsafeIgnoreSidecar bool, additionalDirs []string) (generatedShardIds []uint32, err error) {
+// generateMissingEcFiles takes no block sizes: Reed-Solomon reconstruction is
+// layout-agnostic — it rebuilds a missing shard from the same offsets of the
+// survivors — so the shard layout only ever reaches it through ctx.
+func generateMissingEcFiles(baseFileName string, bufferSize int, ctx *ECContext, unsafeIgnoreSidecar bool, additionalDirs []string) (generatedShardIds []uint32, err error) {
// Pass 1: discover which shards exist and which are missing,
// opening input files but NOT creating output files yet.
@@ -363,6 +453,30 @@ func cleanupRebuildOutputs(outputFiles []*os.File, writePaths []string) {
}
}
+// findVifPath locates the volume's `.vif` for a rebuild: next to the shards
+// first, then in every directory the caller also handed us — the index
+// directory of a split `-dir`/`-dir.idx` layout, and the sibling disks of a
+// multi-disk server, where `.ecx`/`.ecj`/`.vif` may live while this disk holds
+// only shards. Returns the data-base path when nothing exists, so the caller's
+// "not found" handling stays on the canonical name.
+func findVifPath(baseFileName string, additionalDirs []string) string {
+ dataPath := baseFileName + ".vif"
+ candidates := []string{dataPath}
+ base := filepath.Base(baseFileName)
+ for _, dir := range additionalDirs {
+ if dir == "" {
+ continue
+ }
+ candidates = append(candidates, filepath.Join(dir, base)+".vif")
+ }
+ for _, c := range candidates {
+ if _, err := os.Stat(c); err == nil {
+ return c
+ }
+ }
+ return dataPath
+}
+
// loadRebuildSidecar loads and validates the generation-0 checksum sidecar for a
// rebuild. RebuildEcFiles operates on the un-suffixed (generation 0) shard
// names, so only the legacy sidecar is relevant here. Returns BitrotOff when
@@ -381,10 +495,20 @@ func loadRebuildSidecar(baseFileName string, ctx *ECContext, additionalDirs []st
if prot.Generation != 0 {
return nil, BitrotOff
}
- if prot.EcShardConfig == nil ||
- int(prot.EcShardConfig.DataShards) != ctx.DataShards ||
- int(prot.EcShardConfig.ParityShards) != ctx.ParityShards {
- return nil, BitrotOff
+ if prot.EcShardConfig == nil {
+ return nil, BitrotOff // records no geometry -> nothing to contradict
+ }
+ if int(prot.EcShardConfig.DataShards) != ctx.DataShards ||
+ int(prot.EcShardConfig.ParityShards) != ctx.ParityShards ||
+ prot.EcShardConfig.BlockSize != ctx.BlockSize {
+ // Both records describe the same encode, so a disagreement means the
+ // rebuild is about to reconstruct under a geometry the checksums do not
+ // cover. Treating that as "no protection" skipped every input and
+ // output check exactly when they matter most.
+ glog.Warningf("bitrot: sidecar %s records layout %d+%d block %d but the rebuild uses %d+%d block %d",
+ path, prot.EcShardConfig.DataShards, prot.EcShardConfig.ParityShards, prot.EcShardConfig.BlockSize,
+ ctx.DataShards, ctx.ParityShards, ctx.BlockSize)
+ return nil, BitrotInvalid
}
if err := ValidateBitrotManifest(prot, ctx.DataShards, ctx.ParityShards); err != nil {
glog.Warningf("bitrot: sidecar %s manifest invalid: %v", path, err)
diff --git a/weed/storage/erasure_coding/ec_rebuild_safety_test.go b/weed/storage/erasure_coding/ec_rebuild_safety_test.go
index 99b912470b..3ccb4a77bb 100644
--- a/weed/storage/erasure_coding/ec_rebuild_safety_test.go
+++ b/weed/storage/erasure_coding/ec_rebuild_safety_test.go
@@ -5,7 +5,10 @@ import (
"path/filepath"
"testing"
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
+ "github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
)
// readEcxSizeField returns the size field of the .ecx entry for needleId, or
@@ -295,3 +298,101 @@ func TestRebuildEcFiles_CustomRatioRebuildsByteIdentical(t *testing.T) {
t.Errorf("unexpected shard .ec12 for a 9+3 volume")
}
}
+
+// With no .vif, the bitrot sidecar is the surviving record of the volume's
+// geometry. Defaulting to 10+4 and the legacy block layout instead would
+// reconstruct a custom-ratio volume from the wrong matrix — and would make the
+// sidecar look like it disagrees, silently skipping every checksum check.
+func TestRebuildEcFiles_ResolvesGeometryFromSidecarWithoutVif(t *testing.T) {
+ if !BitrotProtectionEnabled {
+ t.Skip("bitrot protection is compiled out")
+ }
+ dir := t.TempDir()
+ base := filepath.Join(dir, "vol")
+ ctx := &ECContext{DataShards: 12, ParityShards: 4}
+ writeRandomDat(t, base, 7000)
+
+ prot, err := WriteEcFiles(base, ctx)
+ if err != nil {
+ t.Fatalf("WriteEcFiles: %v", err)
+ }
+ if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+ t.Fatalf("save sidecar: %v", err)
+ }
+ // No .vif at all: the sidecar has to answer.
+ const dropped = 11
+ want, err := os.ReadFile(base + ToExt(dropped))
+ if err != nil {
+ t.Fatalf("read shard: %v", err)
+ }
+ if err := os.Remove(base + ToExt(dropped)); err != nil {
+ t.Fatalf("remove shard: %v", err)
+ }
+
+ // nil ctx: the production RPC's BackgroundECContext path.
+ if _, err := RebuildEcFiles(base, nil, false); err != nil {
+ t.Fatalf("RebuildEcFiles: %v", err)
+ }
+ got, err := os.ReadFile(base + ToExt(dropped))
+ if err != nil {
+ t.Fatalf("read rebuilt shard: %v", err)
+ }
+ if string(got) != string(want) {
+ t.Errorf("rebuilt shard %d differs; the rebuild used the wrong geometry", dropped)
+ }
+ // The 12+4 shards past the default ratio must still be there.
+ for _, id := range []int{14, 15} {
+ if _, err := os.Stat(base + ToExt(id)); err != nil {
+ t.Errorf("shard %d missing after rebuild: %v", id, err)
+ }
+ }
+}
+
+// A split -dir/-dir.idx layout keeps the .vif with the index, and a multi-disk
+// server may leave this disk holding only shards. Probing just the data base
+// resolves such a volume to the default ratio and the legacy block layout.
+func TestRebuildEcFiles_FindsVifInAnAdditionalDir(t *testing.T) {
+ dataDir := t.TempDir()
+ idxDir := t.TempDir()
+ base := filepath.Join(dataDir, "vol")
+ ctx := &ECContext{DataShards: 12, ParityShards: 4}
+ writeRandomDat(t, base, 7000)
+
+ if _, err := WriteEcFiles(base, ctx); err != nil {
+ t.Fatalf("WriteEcFiles: %v", err)
+ }
+ // The volume's .vif lives with the index, not the shards.
+ if err := volume_info.SaveVolumeInfo(filepath.Join(idxDir, "vol")+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.GetCurrentVersion()),
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: 12, ParityShards: 4, BlockSize: ctx.BlockSize,
+ },
+ }); err != nil {
+ t.Fatalf("save .vif: %v", err)
+ }
+
+ const dropped = 11
+ want, err := os.ReadFile(base + ToExt(dropped))
+ if err != nil {
+ t.Fatalf("read shard: %v", err)
+ }
+ if err := os.Remove(base + ToExt(dropped)); err != nil {
+ t.Fatalf("remove shard: %v", err)
+ }
+
+ if _, err := RebuildEcFiles(base, nil, true, idxDir); err != nil {
+ t.Fatalf("RebuildEcFiles: %v", err)
+ }
+ got, err := os.ReadFile(base + ToExt(dropped))
+ if err != nil {
+ t.Fatalf("read rebuilt shard: %v", err)
+ }
+ if string(got) != string(want) {
+ t.Errorf("rebuilt shard %d differs; the rebuild used the wrong geometry", dropped)
+ }
+ for _, id := range []int{14, 15} {
+ if _, err := os.Stat(base + ToExt(id)); err != nil {
+ t.Errorf("shard %d missing after rebuild: %v", id, err)
+ }
+ }
+}
diff --git a/weed/storage/erasure_coding/ec_roundtrip_test.go b/weed/storage/erasure_coding/ec_roundtrip_test.go
index 8c07a338d1..f70c239708 100644
--- a/weed/storage/erasure_coding/ec_roundtrip_test.go
+++ b/weed/storage/erasure_coding/ec_roundtrip_test.go
@@ -326,7 +326,7 @@ func testDecodeDat(t *testing.T, datSize int64) {
shardFileNames[i] = fmt.Sprintf("%s%s", baseFileName, ctx.ToExt(i))
}
- err = WriteDatFile(decodedBase, datSize, datSize, shardFileNames)
+ err = WriteDatFile(decodedBase, datSize, datSize, shardFileNames, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize)
require.NoError(t, err, "WriteDatFile")
// The atomic publish must rename the temp file away, never leaving it behind.
diff --git a/weed/storage/erasure_coding/ec_uniform_layout_test.go b/weed/storage/erasure_coding/ec_uniform_layout_test.go
new file mode 100644
index 0000000000..215fd7bfaa
--- /dev/null
+++ b/weed/storage/erasure_coding/ec_uniform_layout_test.go
@@ -0,0 +1,250 @@
+package erasure_coding
+
+import (
+ "bytes"
+ "fmt"
+ "math/rand"
+ "os"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/needle"
+ "github.com/seaweedfs/seaweedfs/weed/storage/types"
+ "github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+// legacyShardSize is the padded shard length the two-tier layout produces:
+// whole 1GiB rows, then the remainder in 1MiB rows.
+func legacyShardSize(datFileSize int64, dataShards int) int64 {
+ largeRow := int64(ErasureCodingLargeBlockSize) * int64(dataShards)
+ nLarge := datFileSize / largeRow
+ size := nLarge * ErasureCodingLargeBlockSize
+ if rem := datFileSize - nLarge*largeRow; rem > 0 {
+ smallRow := int64(ErasureCodingSmallBlockSize) * int64(dataShards)
+ size += (rem + smallRow - 1) / smallRow * ErasureCodingSmallBlockSize
+ }
+ return size
+}
+
+// The uniform layout must never change a shard's length, only the byte
+// placement — capacity math and shard-size credibility checks depend on it.
+func TestUniformBlockSizeMatchesLegacyShardSize(t *testing.T) {
+ sizes := []int64{
+ 8, 1000, ErasureCodingSmallBlockSize,
+ 10 * ErasureCodingSmallBlockSize,
+ 10*ErasureCodingSmallBlockSize + 1,
+ 25 * 1024 * 1024,
+ 10 * ErasureCodingLargeBlockSize,
+ 10*ErasureCodingLargeBlockSize + 1,
+ 10*ErasureCodingLargeBlockSize - ErasureCodingSmallBlockSize,
+ 30 * 1024 * 1024 * 1024,
+ 30*1024*1024*1024 + 12345,
+ }
+ for _, datSize := range sizes {
+ for _, ds := range []int{10, 9, 5, 1} {
+ if got, want := UniformBlockSize(datSize, ds), legacyShardSize(datSize, ds); got != want {
+ t.Errorf("UniformBlockSize(%d, %d) = %d, legacy shard size %d", datSize, ds, got, want)
+ }
+ }
+ }
+ if got := UniformBlockSize(0, 10); got != ErasureCodingSmallBlockSize {
+ t.Errorf("UniformBlockSize(0, 10) = %d, want one small block", got)
+ }
+}
+
+// encodeLayoutFixture writes a random .dat and encodes it with the given block
+// sizes, returning the .dat content.
+func encodeLayoutFixture(t *testing.T, baseFileName string, datSize int64, large, small int64, ctx *ECContext) []byte {
+ t.Helper()
+ data := make([]byte, datSize)
+ rand.New(rand.NewSource(datSize)).Read(data)
+ if err := os.WriteFile(baseFileName+".dat", data, 0o644); err != nil {
+ t.Fatalf("write .dat: %v", err)
+ }
+ if _, err := generateEcFiles(baseFileName, 256*1024, large, small, ctx); err != nil {
+ t.Fatalf("generateEcFiles: %v", err)
+ }
+ return data
+}
+
+// readIntervals assembles a byte range from the shard files the way the read
+// path does.
+func readIntervals(t *testing.T, baseFileName string, ctx *ECContext, intervals []Interval) []byte {
+ t.Helper()
+ var assembled []byte
+ for _, iv := range intervals {
+ shardId, shardOffset := iv.ToShardIdAndOffset(ctx.LargeBlockSize(), ctx.SmallBlockSize())
+ f, err := os.Open(baseFileName + ctx.ToExt(int(shardId)))
+ if err != nil {
+ t.Fatalf("open shard %d: %v", shardId, err)
+ }
+ buf := make([]byte, iv.Size)
+ if _, err := f.ReadAt(buf, shardOffset); err != nil {
+ t.Fatalf("read shard %d at %d: %v", shardId, shardOffset, err)
+ }
+ f.Close()
+ assembled = append(assembled, buf...)
+ }
+ return assembled
+}
+
+// Encode 25MB — large enough that the uniform and legacy layouts place bytes
+// differently — under both layouts and verify LocateData maps every probed
+// range back to the original bytes.
+func TestLocateDataMatchesEncoderForBothLayouts(t *testing.T) {
+ const datSize = 25*1024*1024 + 12345
+ dir := t.TempDir()
+
+ uniformCtx := NewDefaultECContext("", 1)
+ uniformCtx.BlockSize = UniformBlockSize(datSize, uniformCtx.DataShards)
+ legacyCtx := NewDefaultECContext("", 2)
+
+ for name, ctx := range map[string]*ECContext{"uniform": uniformCtx, "legacy": legacyCtx} {
+ t.Run(name, func(t *testing.T) {
+ base := fmt.Sprintf("%s/%s", dir, name)
+ data := encodeLayoutFixture(t, base, datSize, ctx.LargeBlockSize(), ctx.SmallBlockSize(), ctx)
+
+ shardSize := datSize / int64(ctx.DataShards)
+ for _, probe := range []struct{ offset, size int64 }{
+ {0, 1000},
+ {ctx.SmallBlockSize() - 100, 200}, // straddles a block boundary
+ {5 * 1024 * 1024, 4 * 1024 * 1024},
+ {datSize - 2000, 2000},
+ } {
+ intervals := LocateData(ctx.LargeBlockSize(), ctx.SmallBlockSize(), shardSize, probe.offset, types.Size(probe.size))
+ got := readIntervals(t, base, ctx, intervals)
+ if !bytes.Equal(got, data[probe.offset:probe.offset+probe.size]) {
+ t.Fatalf("%s layout: bytes at [%d,+%d) do not round-trip", name, probe.offset, probe.size)
+ }
+ }
+
+ // The point of the uniform layout: a 2MB range inside one 3MB
+ // block is a single interval instead of three 1MB stripes.
+ intervals := LocateData(ctx.LargeBlockSize(), ctx.SmallBlockSize(), shardSize, 6*1024*1024+512*1024, types.Size(2*1024*1024))
+ if name == "uniform" && len(intervals) != 1 {
+ t.Errorf("uniform layout reads 2MB in %d intervals, want 1", len(intervals))
+ }
+ if name == "legacy" && len(intervals) != 3 {
+ t.Errorf("legacy layout reads 2MB in %d intervals, want 3", len(intervals))
+ }
+ })
+ }
+}
+
+// Encode under both layouts and verify WriteDatFile reconstructs the original
+// .dat byte for byte — and that decoding a uniform volume with the legacy
+// geometry does NOT, i.e. the recorded block size is load-bearing.
+func TestUniformDecodeRoundTrip(t *testing.T) {
+ const datSize = 25 * 1024 * 1024
+ dir := t.TempDir()
+ base := dir + "/1"
+
+ ctx := NewDefaultECContext("", 1)
+ ctx.BlockSize = UniformBlockSize(datSize, ctx.DataShards)
+ data := encodeLayoutFixture(t, base, datSize, ctx.BlockSize, ctx.BlockSize, ctx)
+
+ shardFileNames := make([]string, ctx.DataShards)
+ for i := range shardFileNames {
+ shardFileNames[i] = base + ctx.ToExt(i)
+ }
+
+ if err := WriteDatFile(dir+"/decoded", datSize, datSize, shardFileNames, ctx.LargeBlockSize(), ctx.SmallBlockSize()); err != nil {
+ t.Fatalf("WriteDatFile: %v", err)
+ }
+ decoded, err := os.ReadFile(dir + "/decoded.dat")
+ if err != nil {
+ t.Fatalf("read decoded: %v", err)
+ }
+ if !bytes.Equal(decoded, data) {
+ t.Fatal("uniform decode with recorded geometry does not round-trip")
+ }
+
+ if err := WriteDatFile(dir+"/wrong", datSize, datSize, shardFileNames, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize); err != nil {
+ t.Fatalf("WriteDatFile legacy geometry: %v", err)
+ }
+ wrong, err := os.ReadFile(dir + "/wrong.dat")
+ if err != nil {
+ t.Fatalf("read wrong: %v", err)
+ }
+ if bytes.Equal(wrong, data) {
+ t.Fatal("decoding a uniform volume with legacy geometry should scramble it; the layouts did not diverge")
+ }
+}
+
+// Mount fixtures through NewEcVolume and read a large extent back through
+// LocateEcShardNeedleInterval, proving the .vif block size steers the read
+// path: a legacy .vif (no block size) keeps the legacy interpretation, a
+// uniform .vif reads uniform shards.
+func TestEcVolumeGeometryFromVif(t *testing.T) {
+ const datSize = 25 * 1024 * 1024
+
+ for _, layout := range []string{"legacy", "uniform"} {
+ t.Run(layout, func(t *testing.T) {
+ dir := t.TempDir()
+ vid := needle.VolumeId(7)
+ base := EcShardFileName("", dir, int(vid))
+
+ ctx := NewDefaultECContext("", vid)
+ if layout == "uniform" {
+ ctx.BlockSize = UniformBlockSize(datSize, ctx.DataShards)
+ }
+ data := encodeLayoutFixture(t, base, datSize, ctx.LargeBlockSize(), ctx.SmallBlockSize(), ctx)
+
+ if err := os.WriteFile(base+".ecx", nil, 0o644); err != nil {
+ t.Fatalf("write .ecx: %v", err)
+ }
+ if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.GetCurrentVersion()),
+ DatFileSize: datSize,
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: uint32(ctx.DataShards),
+ ParityShards: uint32(ctx.ParityShards),
+ BlockSize: ctx.BlockSize,
+ },
+ }); err != nil {
+ t.Fatalf("save .vif: %v", err)
+ }
+
+ ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", vid)
+ if err != nil {
+ t.Fatalf("NewEcVolume: %v", err)
+ }
+ defer ev.Close()
+ if ev.ECContext.BlockSize != ctx.BlockSize {
+ t.Fatalf("loaded BlockSize = %d, want %d", ev.ECContext.BlockSize, ctx.BlockSize)
+ }
+ for i := 0; i < ctx.DataShards; i++ {
+ shard, err := NewEcVolumeShard(types.HardDriveType, dir, "", vid, ShardId(i))
+ if err != nil {
+ t.Fatalf("NewEcVolumeShard %d: %v", i, err)
+ }
+ ev.AddEcVolumeShard(shard)
+ }
+
+ // Sweep a large extent through the volume's own interval mapping.
+ // LocateEcShardNeedleInterval expands the size by the needle
+ // overhead, so compare only the probed prefix.
+ const probeSize = datSize - 64*1024
+ intervals := ev.LocateEcShardNeedleInterval(needle.GetCurrentVersion(), 0, types.Size(probeSize))
+ var assembled []byte
+ for _, iv := range intervals {
+ shardId, shardOffset := ev.IntervalToShardIdAndOffset(iv)
+ shard, found := ev.FindEcVolumeShard(shardId)
+ if !found {
+ t.Fatalf("shard %d not mounted", shardId)
+ }
+ buf := make([]byte, iv.Size)
+ if _, err := shard.ReadAt(buf, shardOffset); err != nil {
+ t.Fatalf("read shard %d at %d: %v", shardId, shardOffset, err)
+ }
+ assembled = append(assembled, buf...)
+ }
+ if len(assembled) < probeSize {
+ t.Fatalf("assembled %d bytes, want at least %d", len(assembled), probeSize)
+ }
+ if !bytes.Equal(assembled[:probeSize], data[:probeSize]) {
+ t.Fatalf("%s layout: full-extent read through the %s .vif does not match the .dat", layout, layout)
+ }
+ })
+ }
+}
diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go
index 8bb4a4e8f0..cb64639489 100644
--- a/weed/storage/erasure_coding/ec_volume.go
+++ b/weed/storage/erasure_coding/ec_volume.go
@@ -173,7 +173,16 @@ func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection
}
}
ev.Version = needle.Version3
- if volumeInfo, _, found, _ := volume_info.MaybeLoadVolumeInfo(vifFileName); found {
+ // A present-but-unreadable or malformed .vif FAILS the mount: every new
+ // encode records a positive uniform block size there, and defaulting to
+ // the legacy layout would serve those shards with the wrong offset math.
+ // Absent stays legal — legacy volumes predate the sidecar.
+ volumeInfo, _, found, vifErr := volume_info.MaybeLoadVolumeInfo(vifFileName)
+ if vifErr != nil {
+ ev.Close()
+ return nil, fmt.Errorf("ec volume %d: load %s: %w", vid, vifFileName, vifErr)
+ }
+ if found {
ev.Version = needle.Version(volumeInfo.Version)
ev.datFileSize = volumeInfo.DatFileSize
ev.ExpireAtSec = volumeInfo.ExpireAtSec
@@ -184,21 +193,56 @@ func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection
ps := int(volumeInfo.EcShardConfig.ParityShards)
ev.EncodeTsNs = volumeInfo.EcShardConfig.GetEncodeTsNs()
- // Validate shard counts to prevent zero or invalid values
- if ds <= 0 || ps <= 0 || ds+ps > MaxShardCount {
- glog.Warningf("Invalid EC config in VolumeInfo for volume %d (data=%d, parity=%d), using defaults", vid, ds, ps)
- ev.ECContext = NewDefaultECContext(collection, vid)
+ // A config that is PRESENT but records an impossible ratio is not
+ // a volume to fall back on: substituting the default 10+4 with the
+ // legacy layout would read uniform shards with the wrong offset
+ // math and answer with the wrong bytes. Only an ENTIRELY absent
+ // config means "this predates the record", which the else-branch
+ // below serves with the legacy defaults.
+ if !ValidEcShardCounts(volumeInfo.EcShardConfig.DataShards, volumeInfo.EcShardConfig.ParityShards) {
+ ev.Close()
+ return nil, fmt.Errorf("ec volume %d: %s records invalid shard counts %d+%d",
+ vid, vifFileName, volumeInfo.EcShardConfig.DataShards, volumeInfo.EcShardConfig.ParityShards)
+ } else if blockErr := ValidateBlockSize(volumeInfo.EcShardConfig.GetBlockSize()); blockErr != nil {
+ // A recorded block size that no encoder could have produced maps
+ // every read to the wrong shard offset. Refuse the mount rather
+ // than serve those bytes or silently pick a layout.
+ ev.Close()
+ return nil, fmt.Errorf("ec volume %d: %s: %w", vid, vifFileName, blockErr)
} else {
ev.ECContext = &ECContext{
Collection: collection,
VolumeId: vid,
DataShards: ds,
ParityShards: ps,
+ BlockSize: volumeInfo.EcShardConfig.GetBlockSize(),
}
glog.V(1).Infof("Loaded EC config from VolumeInfo for volume %d: %s", vid, ev.ECContext.String())
}
} else {
- ev.ECContext = NewDefaultECContext(collection, vid)
+ // A vif that carries no ecShardConfig answers nothing about the
+ // layout — it is no more informative than an absent one, so it
+ // must not skip the sidecar. Going straight to the defaults here
+ // read a uniform volume with the legacy offset math.
+ cfg, sidecarFound, sidecarErr := layoutFromSidecar(dataBaseFileName, indexBaseFileName)
+ if sidecarErr != nil {
+ ev.Close()
+ return nil, fmt.Errorf("ec volume %d: %s records no EC config and the bitrot sidecar cannot establish the layout: %w", vid, vifFileName, sidecarErr)
+ }
+ if sidecarFound {
+ ev.ECContext = &ECContext{
+ Collection: collection,
+ VolumeId: vid,
+ DataShards: int(cfg.GetDataShards()),
+ ParityShards: int(cfg.GetParityShards()),
+ BlockSize: cfg.GetBlockSize(),
+ }
+ ev.EncodeTsNs = cfg.GetEncodeTsNs()
+ glog.V(0).Infof("ec volume %d: .vif records no EC config; took it from the bitrot sidecar: %s",
+ vid, ev.ECContext.String())
+ } else {
+ ev.ECContext = NewDefaultECContext(collection, vid)
+ }
}
} else {
// Don't fabricate a stub .vif here: a version-only stub implies the
@@ -207,14 +251,48 @@ func NewEcVolume(diskType types.DiskType, dir string, dirIdx string, collection
// mistake for an authoritative config. Mount with in-memory defaults and
// leave the real .vif to the encoder or a recovery tool (the Rust volume
// server already behaves this way).
- glog.Warningf("vif file not found, using defaults, volumeId:%d, filename:%s", vid, vifFileName)
+ //
+ // The bitrot sidecar records the same EC config at encode time, so when
+ // it is present it answers the layout question the missing .vif cannot:
+ // defaulting a uniform-layout volume to the legacy block sizes maps
+ // every read to the wrong shard offset. `weed fix -ecx` reads the
+ // sidecar for the same reason.
ev.ECContext = NewDefaultECContext(collection, vid)
+ cfg, sidecarFound, sidecarErr := layoutFromSidecar(dataBaseFileName, indexBaseFileName)
+ if sidecarErr != nil {
+ // With no .vif the sidecar is the ONLY record of this volume's
+ // layout. Present but unusable is not "assume legacy" — that
+ // answers reads with the wrong shard offsets, which is worse than
+ // not answering at all.
+ ev.Close()
+ return nil, fmt.Errorf("ec volume %d: no .vif and the bitrot sidecar cannot establish the layout: %w", vid, sidecarErr)
+ }
+ if sidecarFound {
+ ev.ECContext = &ECContext{
+ Collection: collection,
+ VolumeId: vid,
+ DataShards: int(cfg.GetDataShards()),
+ ParityShards: int(cfg.GetParityShards()),
+ BlockSize: cfg.GetBlockSize(),
+ }
+ ev.EncodeTsNs = cfg.GetEncodeTsNs()
+ glog.V(0).Infof("ec volume %d: .vif missing; took EC config from the bitrot sidecar: %s",
+ vid, ev.ECContext.String())
+ } else {
+ // Only now are the defaults what the volume actually mounted on;
+ // logging this after the sidecar answered would send an operator
+ // triaging wrong bytes after the legacy layout instead.
+ glog.Warningf("vif file not found, using defaults, volumeId:%d, filename:%s", vid, vifFileName)
+ }
}
ev.ShardLocations = make(map[ShardId][]pb.ServerAddress)
// Load the active-generation bitrot checksum sidecar (optional).
- ev.loadActiveBitrotSidecar()
+ if err := ev.loadActiveBitrotSidecar(); err != nil {
+ ev.Close()
+ return nil, err
+ }
return
}
@@ -528,11 +606,17 @@ func (ev *EcVolume) LocateEcShardNeedleInterval(version needle.Version, offset i
shardSize = shard.ecdFileSize - 1
}
// calculate the locations in the ec shards
- intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shardSize, offset, types.Size(needle.GetActualSize(size, version)))
+ intervals = LocateData(ev.ECContext.LargeBlockSize(), ev.ECContext.SmallBlockSize(), shardSize, offset, types.Size(needle.GetActualSize(size, version)))
return
}
+// IntervalToShardIdAndOffset resolves an interval against this volume's shard
+// block layout.
+func (ev *EcVolume) IntervalToShardIdAndOffset(interval Interval) (ShardId, int64) {
+ return interval.ToShardIdAndOffset(ev.ECContext.LargeBlockSize(), ev.ECContext.SmallBlockSize())
+}
+
func (ev *EcVolume) FindNeedleFromEcx(needleId types.NeedleId) (offset types.Offset, size types.Size, err error) {
offset, size, err = SearchNeedleFromSortedIndex(ev.ecxFile, ev.ecxFileSize, needleId, nil)
if err != nil {
diff --git a/weed/storage/erasure_coding/ec_volume_scrub.go b/weed/storage/erasure_coding/ec_volume_scrub.go
index 8cb33297eb..40457f58e2 100644
--- a/weed/storage/erasure_coding/ec_volume_scrub.go
+++ b/weed/storage/erasure_coding/ec_volume_scrub.go
@@ -238,7 +238,7 @@ func (ecv *EcVolume) ScrubLocal() (int64, []*volume_server_pb.EcShardInfo, []err
localShardIds := []ShardId{}
for i, iv := range locations {
- sid, soffset := iv.ToShardIdAndOffset(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize)
+ sid, soffset := ecv.IntervalToShardIdAndOffset(iv)
ssize := int64(iv.Size.Raw())
shard, found := ecv.FindEcVolumeShard(sid)
diff --git a/weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go b/weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go
new file mode 100644
index 0000000000..34e992416e
--- /dev/null
+++ b/weed/storage/erasure_coding/ec_volume_sidecar_layout_test.go
@@ -0,0 +1,246 @@
+package erasure_coding
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/needle"
+ "github.com/seaweedfs/seaweedfs/weed/storage/types"
+ "github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+func uniformSidecar(t *testing.T, base string, ds, ps uint32, blockSize int64) {
+ t.Helper()
+ prot := &volume_server_pb.EcBitrotProtection{
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: 0,
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: ds, ParityShards: ps, BlockSize: blockSize,
+ },
+ }
+ if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+ t.Fatalf("seed .ecsum: %v", err)
+ }
+}
+
+// The sidecar records the layout at encode time, so it answers what a .vif
+// cannot. A .vif that merely omits ecShardConfig is no more informative than an
+// absent one — going straight to the defaults reads a uniform volume with the
+// legacy offset math and returns the wrong bytes.
+func TestNewEcVolumeTakesLayoutFromSidecarWhenVifHasNoConfig(t *testing.T) {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "1")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ // A config-free .vif: version and dat size only, as a legacy encode left it.
+ if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ }); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+ uniformSidecar(t, base, 12, 4, 3*1024*1024)
+
+ ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+ if err != nil {
+ t.Fatalf("mount: %v", err)
+ }
+ defer ev.Close()
+ if got := ev.ECContext; got.DataShards != 12 || got.ParityShards != 4 || got.BlockSize != 3*1024*1024 {
+ t.Errorf("layout = %d+%d block %d, want 12+4 block 3MiB", got.DataShards, got.ParityShards, got.BlockSize)
+ }
+}
+
+// A split -dir/-dir.idx layout keeps the sidecar with the INDEX. Probing only
+// the data base reports "absent", and absent is the one answer that selects the
+// legacy layout.
+func TestNewEcVolumeFindsSidecarInTheIndexDirectory(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ writeVif bool
+ }{
+ {name: "no vif at all"},
+ {name: "vif without ec config", writeVif: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ dataDir, idxDir := t.TempDir(), t.TempDir()
+ idxBase := filepath.Join(idxDir, "2")
+ if err := os.WriteFile(idxBase+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ if tc.writeVif {
+ if err := volume_info.SaveVolumeInfo(idxBase+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ }); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+ }
+ uniformSidecar(t, idxBase, 12, 4, 3*1024*1024)
+
+ ev, err := NewEcVolume(types.HardDriveType, dataDir, idxDir, "", needle.VolumeId(2))
+ if err != nil {
+ t.Fatalf("mount: %v", err)
+ }
+ defer ev.Close()
+ if got := ev.ECContext; got.DataShards != 12 || got.ParityShards != 4 || got.BlockSize != 3*1024*1024 {
+ t.Errorf("layout = %d+%d block %d, want 12+4 block 3MiB", got.DataShards, got.ParityShards, got.BlockSize)
+ }
+ })
+ }
+}
+
+// Absence stays legal — a volume encoded before either record exists is a
+// genuine legacy volume, and only that case may select the legacy layout.
+func TestNewEcVolumeConfigFreeVifWithNoSidecarUsesDefaults(t *testing.T) {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "3")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ }); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+ ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(3))
+ if err != nil {
+ t.Fatalf("mount: %v", err)
+ }
+ defer ev.Close()
+ if got := ev.ECContext; got.DataShards != DataShardsCount || got.ParityShards != ParityShardsCount || got.BlockSize != 0 {
+ t.Errorf("layout = %d+%d block %d, want the legacy defaults", got.DataShards, got.ParityShards, got.BlockSize)
+ }
+}
+
+// Present but unusable must fail the mount rather than default, in the
+// config-free-vif branch exactly as in the absent-vif one.
+func TestNewEcVolumeConfigFreeVifWithUnusableSidecarFails(t *testing.T) {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "4")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ }); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+ // Stamped for another generation: not a record of these shards.
+ if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), &volume_server_pb.EcBitrotProtection{
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: 5,
+ EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 12, ParityShards: 4},
+ }); err != nil {
+ t.Fatalf("seed .ecsum: %v", err)
+ }
+ if _, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(4)); err == nil {
+ t.Fatal("an unusable sole layout record must fail the mount")
+ }
+}
+
+// Startup mirroring gives each shard-bearing disk its own .ecx/.ecj/.vif but
+// deliberately not the .ecsum, and a repair delivers exactly one copy. A
+// runtime restricted to its own two directories therefore reports no
+// protection no matter how often it reloads — so the resolution has to reach
+// the sibling disk that actually received the manifest.
+func TestReloadBitrotSidecarFindsItOnASiblingDisk(t *testing.T) {
+ mine, sibling := t.TempDir(), t.TempDir()
+ myBase := filepath.Join(mine, "7")
+ if err := os.WriteFile(myBase+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ if err := volume_info.SaveVolumeInfo(myBase+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: 10, ParityShards: 4, BlockSize: 0,
+ },
+ }); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+
+ ev, err := NewEcVolume(types.HardDriveType, mine, mine, "", needle.VolumeId(7))
+ if err != nil {
+ t.Fatalf("mount: %v", err)
+ }
+ defer ev.Close()
+ if got := ev.bitrotStatus; got != BitrotOff {
+ t.Fatalf("precondition: status = %v, want BitrotOff before the delivery", got)
+ }
+
+ // The delivery lands the manifest on the sibling disk only.
+ prot := &volume_server_pb.EcBitrotProtection{
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: 0,
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: 10, ParityShards: 4, BlockSize: 0,
+ },
+ }
+ for i := 0; i < 14; i++ {
+ prot.Shards = append(prot.Shards, &volume_server_pb.EcShardChecksums{
+ ShardId: uint32(i), CoveredSize: 1, BlockCrc32C: make([]byte, 4),
+ })
+ }
+ if err := SaveBitrotSidecar(BitrotSidecarPath(filepath.Join(sibling, "7"), 0), prot); err != nil {
+ t.Fatalf("seed sibling .ecsum: %v", err)
+ }
+
+ // Reloading against only its own directories still finds nothing.
+ ev.ReloadBitrotSidecar()
+ if got := ev.bitrotStatus; got != BitrotOff {
+ t.Errorf("own-directories reload: status = %v, want BitrotOff", got)
+ }
+
+ ev.ReloadBitrotSidecar(mine, sibling)
+ if got := ev.bitrotStatus; got != BitrotOn {
+ t.Errorf("cross-disk reload: status = %v, want BitrotOn", got)
+ }
+}
+
+// "Does this volume already have a manifest" cannot be answered from one base
+// name. The rebuild's opportunistic backfill asks it before writing a TOFU
+// baseline, and a false "no" there writes a second sidecar at the data base
+// that then shadows the real one — blessing whatever the shards currently say.
+func TestFindBitrotSidecarSearchesEveryCandidate(t *testing.T) {
+ dataDir, idxDir, sibling := t.TempDir(), t.TempDir(), t.TempDir()
+ dataBase := filepath.Join(dataDir, "8")
+ idxBase := filepath.Join(idxDir, "8")
+ siblingBase := filepath.Join(sibling, "8")
+
+ if got := FindBitrotSidecar(0, dataBase, idxBase, sibling); got != "" {
+ t.Errorf("no sidecar anywhere should answer empty, got %q", got)
+ }
+
+ // Each location on its own: any one of them is enough to answer "yes".
+ for _, tc := range []struct{ name, base string }{
+ {"data directory", dataBase},
+ {"index directory", idxBase},
+ {"sibling disk", siblingBase},
+ } {
+ want := BitrotSidecarPath(tc.base, 0)
+ if err := os.WriteFile(want, []byte("x"), 0644); err != nil {
+ t.Fatalf("%s: seed: %v", tc.name, err)
+ }
+ if got := FindBitrotSidecar(0, dataBase, idxBase, sibling); got != want {
+ t.Errorf("%s: got %q, want %q", tc.name, got, want)
+ }
+ if err := os.Remove(want); err != nil {
+ t.Fatalf("%s: cleanup: %v", tc.name, err)
+ }
+ }
+
+ // With copies everywhere the data base wins, which is the order every
+ // resolver uses — so a stray copy there shadows the others.
+ for _, base := range []string{siblingBase, idxBase, dataBase} {
+ if err := os.WriteFile(BitrotSidecarPath(base, 0), []byte("x"), 0644); err != nil {
+ t.Fatalf("seed %s: %v", base, err)
+ }
+ }
+ if got, want := FindBitrotSidecar(0, dataBase, idxBase, sibling), BitrotSidecarPath(dataBase, 0); got != want {
+ t.Errorf("precedence: got %q, want the data base %q", got, want)
+ }
+}
diff --git a/weed/storage/erasure_coding/ec_volume_vif_test.go b/weed/storage/erasure_coding/ec_volume_vif_test.go
new file mode 100644
index 0000000000..e054d7612a
--- /dev/null
+++ b/weed/storage/erasure_coding/ec_volume_vif_test.go
@@ -0,0 +1,187 @@
+package erasure_coding
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/needle"
+ "github.com/seaweedfs/seaweedfs/weed/storage/types"
+ "github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
+)
+
+// A present-but-malformed .vif must FAIL the mount: every new encode
+// records a positive uniform block size there, and silently defaulting
+// to the legacy layout would serve those shards with the wrong offset
+// math. Absence stays legal — legacy volumes predate the sidecar.
+func TestNewEcVolumeFailsOnMalformedVif(t *testing.T) {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "1")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ if err := os.WriteFile(base+".vif", []byte("not json"), 0644); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+ _, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+ if err == nil {
+ t.Fatal("mount over a malformed .vif must fail")
+ }
+ if !strings.Contains(err.Error(), ".vif") {
+ t.Errorf("error should name the vif: %v", err)
+ }
+}
+
+func TestNewEcVolumeAbsentVifMountsWithDefaults(t *testing.T) {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "1")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+ if err != nil {
+ t.Fatalf("legacy mount without .vif must succeed: %v", err)
+ }
+ defer ev.Close()
+ if ev.ECContext.BlockSize != 0 {
+ t.Errorf("legacy mount must use the legacy layout: BlockSize=%d", ev.ECContext.BlockSize)
+ }
+}
+
+// A block size no encoder could have produced (negative, or not a whole
+// number of small blocks) maps every read to the wrong shard offset, so the
+// mount refuses it rather than serving those bytes.
+func TestNewEcVolumeRejectsImplausibleBlockSize(t *testing.T) {
+ for _, blockSize := range []int64{1, -1, 3*1024*1024 + 1} {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "1")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: 10, ParityShards: 4, BlockSize: blockSize,
+ },
+ }); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+ if _, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1)); err == nil {
+ t.Errorf("block size %d must fail the mount", blockSize)
+ }
+ }
+}
+
+func TestNewEcVolumeAcceptsAlignedBlockSize(t *testing.T) {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "1")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: 10, ParityShards: 4, BlockSize: 3 * 1024 * 1024,
+ },
+ }); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+ ev, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+ if err != nil {
+ t.Fatalf("aligned block size must mount: %v", err)
+ }
+ defer ev.Close()
+ if ev.ECContext.BlockSize != 3*1024*1024 {
+ t.Errorf("BlockSize = %d, want 3MiB", ev.ECContext.BlockSize)
+ }
+}
+
+// The .vif and the .ecsum both record the layout their generation was encoded
+// with. When a generation-matching sidecar disagrees, one of them is wrong and
+// reads through the other land at the wrong shard offsets — so the mount must
+// refuse rather than quietly drop to unprotected reads.
+func TestNewEcVolumeRejectsSidecarGeometryDisagreement(t *testing.T) {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "1")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("seed .ecx: %v", err)
+ }
+ if err := volume_info.SaveVolumeInfo(base+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.Version3),
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: 10, ParityShards: 4, BlockSize: 0, // says legacy
+ },
+ }); err != nil {
+ t.Fatalf("seed .vif: %v", err)
+ }
+ // The sidecar for the same generation says uniform.
+ prot := &volume_server_pb.EcBitrotProtection{
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: 0,
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: 10, ParityShards: 4, BlockSize: 3 * 1024 * 1024,
+ },
+ }
+ if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+ t.Fatalf("seed .ecsum: %v", err)
+ }
+
+ _, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1))
+ if err == nil {
+ t.Fatal("a generation-matching layout disagreement must fail the mount")
+ }
+ if !strings.Contains(err.Error(), "block") {
+ t.Errorf("the error should name the disagreement: %v", err)
+ }
+}
+
+// With no .vif the bitrot sidecar is the ONLY record of the volume's layout.
+// Present but unusable — unreadable, stamped for another generation, or
+// carrying an incomplete ratio — must fail the mount: answering reads from a
+// guessed layout returns wrong bytes rather than none. Genuine absence stays
+// legal, and is covered by TestNewEcVolumeAbsentVifMountsWithDefaults.
+func TestNewEcVolumeRejectsUnusableSoleSidecar(t *testing.T) {
+ seed := map[string]*volume_server_pb.EcBitrotProtection{
+ "another generation": {
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: 5,
+ EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10, ParityShards: 4},
+ },
+ "no ec config": {
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: 0,
+ },
+ "incomplete ratio": {
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: 0,
+ EcShardConfig: &volume_server_pb.EcShardConfig{DataShards: 10},
+ },
+ "invalid block size": {
+ Algorithm: volume_server_pb.ChecksumAlgorithm_CHECKSUM_CRC32C,
+ BlockSize: uint32(BitrotBlockSize),
+ Generation: 0,
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: 10, ParityShards: 4, BlockSize: 3*1024*1024 + 1,
+ },
+ },
+ }
+ for name, prot := range seed {
+ dir := t.TempDir()
+ base := filepath.Join(dir, "1")
+ if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
+ t.Fatalf("%s: seed .ecx: %v", name, err)
+ }
+ if err := SaveBitrotSidecar(BitrotSidecarPath(base, 0), prot); err != nil {
+ t.Fatalf("%s: seed .ecsum: %v", name, err)
+ }
+ if _, err := NewEcVolume(types.HardDriveType, dir, dir, "", needle.VolumeId(1)); err == nil {
+ t.Errorf("%s: an unusable sole layout record must fail the mount", name)
+ }
+ }
+}
diff --git a/weed/storage/erasure_coding/verification.go b/weed/storage/erasure_coding/verification.go
index ac6fa6ecb9..76c52a2e13 100644
--- a/weed/storage/erasure_coding/verification.go
+++ b/weed/storage/erasure_coding/verification.go
@@ -14,6 +14,11 @@ import (
type ServerShardInventory struct {
Bits ShardBits
QueryError error
+ // BlockSize is the shard block layout this holder reports for the volume —
+ // the one it will serve reads through. A binary predating the uniform
+ // layout drops the field off the wire and reports 0, which is how
+ // RequireAgreedBlockLayout tells such a holder from one that agrees.
+ BlockSize int64
}
// Query errors are recorded per-server and treated as zero shards rather
@@ -50,6 +55,7 @@ func VerifyShardsAcrossServers(ctx context.Context, volumeID uint32,
}
inv.Bits = inv.Bits.Set(ShardId(s.ShardId))
}
+ inv.BlockSize = resp.GetEcShardConfig().GetBlockSize()
return nil
})
if callErr != nil {
@@ -99,6 +105,39 @@ func RequireRecoverableShardSet(volumeID uint32, shardsPresent ShardBits, dataSh
volumeID, totalShards-len(missing), totalShards, dataShards, missing)
}
+// RequireAgreedBlockLayout gates source-volume deletion on every holder that
+// answered agreeing with the layout the shards were encoded under.
+//
+// The uniform block layout lives in a `.vif` field that older volume servers
+// have never heard of: such a server parses the file, silently discards the
+// unknown field, and mounts the volume on the legacy 1GiB/1MiB striping. Its
+// reads then land at the wrong shard offsets and return wrong bytes, and
+// nothing else in the encode path notices — the shard files are the same
+// length under either layout. Asking each holder which layout it is serving is
+// the one question that separates the two, and asking it here means the answer
+// arrives while the source .dat is still on disk.
+//
+// Holders that could not be reached, or that report no shard of this volume,
+// are skipped: RequireRecoverableShardSet already covers a missing holder, and
+// one that answered nothing has promised nothing.
+func RequireAgreedBlockLayout(volumeID uint32, encodedBlockSize int64, perServer map[string]ServerShardInventory) error {
+ disagreeing := make([]string, 0, len(perServer))
+ for server, inv := range perServer {
+ if inv.QueryError != nil || inv.Bits.Count() == 0 {
+ continue
+ }
+ if inv.BlockSize != encodedBlockSize {
+ disagreeing = append(disagreeing, fmt.Sprintf("%s serves block size %d", server, inv.BlockSize))
+ }
+ }
+ if len(disagreeing) == 0 {
+ return nil
+ }
+ sort.Strings(disagreeing)
+ return fmt.Errorf("volume %d was encoded with shard block size %d but %v; upgrade every volume server to a build that understands the uniform shard block layout before encoding",
+ volumeID, encodedBlockSize, disagreeing)
+}
+
func SummarizeShardInventory(perServer map[string]ServerShardInventory) string {
servers := make([]string, 0, len(perServer))
for s := range perServer {
diff --git a/weed/storage/store_ec.go b/weed/storage/store_ec.go
index 0abd9fc43e..9ab91b7355 100644
--- a/weed/storage/store_ec.go
+++ b/weed/storage/store_ec.go
@@ -345,6 +345,49 @@ func (s *Store) FindEcVolumeWithShard(vid needle.VolumeId, shardId erasure_codin
return nil, nil, false
}
+// EcMetadataDirs lists every directory on this server that could hold an EC
+// volume's metadata — each disk's data and index directory. Startup mirroring
+// gives each shard-bearing disk its own .ecx/.ecj/.vif, but the checksum
+// sidecar is not mirrored and a repair delivers exactly one copy, so a runtime
+// looking only at its own two directories cannot see it. Handing this list to
+// the sidecar resolution keeps one authoritative copy reachable from every
+// runtime rather than duplicating a file that is rewritten as shards are
+// repaired and generations published.
+func (s *Store) EcMetadataDirs() []string {
+ var dirs []string
+ appendDir := func(dir string) {
+ if dir == "" {
+ return
+ }
+ for _, existing := range dirs {
+ if existing == dir {
+ return
+ }
+ }
+ dirs = append(dirs, dir)
+ }
+ for _, location := range s.Locations {
+ appendDir(location.Directory)
+ appendDir(location.IdxDirectory)
+ }
+ return dirs
+}
+
+// FindAllEcVolumes returns every per-disk *EcVolume the store maps for vid. A vid
+// can mount on N disks as N distinct runtimes, and the first-match FindEcVolume
+// hides the siblings — so anything that has to reach the whole volume, rather
+// than any one runtime of it, iterates this instead. Order mirrors the
+// deterministic s.Locations order; nil when no disk holds the vid.
+func (s *Store) FindAllEcVolumes(vid needle.VolumeId) []*erasure_coding.EcVolume {
+ var evs []*erasure_coding.EcVolume
+ for _, location := range s.Locations {
+ if ev, found := location.FindEcVolume(vid); found {
+ evs = append(evs, ev)
+ }
+ }
+ return evs
+}
+
func (s *Store) FindEcVolume(vid needle.VolumeId) (*erasure_coding.EcVolume, bool) {
for _, location := range s.Locations {
if s, found := location.FindEcVolume(vid); found {
@@ -436,10 +479,6 @@ func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle, onReadS
return 0, fmt.Errorf("ec shard %d not found", vid)
}
-func (s *Store) IntervalToShardIdAndOffset(iv erasure_coding.Interval) (erasure_coding.ShardId, int64) {
- return iv.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
-}
-
var (
// ecRecoverBudget bounds the interval-sized buffers EC recovery holds across
// all concurrent reads: a peer that is slow to fail keeps a whole fan-out of
@@ -507,7 +546,7 @@ func (s *Store) readEcShardIntervals(needleId types.NeedleId, ecVolume *erasure_
// readOneEcShardInterval fills data, which must be interval.Size long.
func (s *Store) readOneEcShardInterval(needleId types.NeedleId, ecVolume *erasure_coding.EcVolume, interval erasure_coding.Interval, data []byte) (is_deleted bool, err error) {
- shardId, actualOffset := s.IntervalToShardIdAndOffset(interval)
+ shardId, actualOffset := ecVolume.IntervalToShardIdAndOffset(interval)
// try local read
err = s.readLocalEcShardInterval(ecVolume, shardId, data, actualOffset)
diff --git a/weed/storage/store_ec_delete.go b/weed/storage/store_ec_delete.go
index da20ffd529..970967cb00 100644
--- a/weed/storage/store_ec_delete.go
+++ b/weed/storage/store_ec_delete.go
@@ -52,7 +52,7 @@ func (s *Store) doDeleteNeedleFromAtLeastOneRemoteEcShards(ecVolume *erasure_cod
return erasure_coding.NotFoundError
}
- primaryShardId, _ := intervals[0].ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
+ primaryShardId, _ := ecVolume.IntervalToShardIdAndOffset(intervals[0])
// Normal path: delete on exactly one node holding the primary data shard.
err = s.doDeleteNeedleFromRemoteEcShardServers(primaryShardId, ecVolume, needleId)
diff --git a/weed/storage/store_ec_interval_read_test.go b/weed/storage/store_ec_interval_read_test.go
index 8dba1bb03e..dedb57a335 100644
--- a/weed/storage/store_ec_interval_read_test.go
+++ b/weed/storage/store_ec_interval_read_test.go
@@ -2,17 +2,112 @@ package storage
import (
"bytes"
+ "math/rand"
+ "os"
"testing"
+ "time"
+ "github.com/seaweedfs/seaweedfs/weed/pb"
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
+ "github.com/seaweedfs/seaweedfs/weed/storage/super_block"
+ "github.com/seaweedfs/seaweedfs/weed/storage/types"
+ "github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
)
+// encodeAndMountEcVolume writes needles into a fresh volume, EC-encodes it
+// (uniform layout), stamps the .vif the way VolumeEcShardsGenerate does, and
+// mounts every shard on the store.
+func encodeAndMountEcVolume(t *testing.T, store *Store, vid needle.VolumeId, needles []*needle.Needle) *erasure_coding.EcVolume {
+ t.Helper()
+ dir := store.Locations[0].Directory
+
+ v, err := NewVolume(dir, dir, "", vid, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
+ if err != nil {
+ t.Fatalf("new volume: %v", err)
+ }
+ for _, n := range needles {
+ if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil {
+ t.Fatalf("write needle: %v", err)
+ }
+ }
+ baseFileName := v.DataFileName()
+ v.Close()
+
+ datSize, err := os.Stat(baseFileName + ".dat")
+ if err != nil {
+ t.Fatalf("stat .dat: %v", err)
+ }
+ ecCtx := erasure_coding.NewDefaultECContext("", vid)
+ if _, err := erasure_coding.WriteEcFiles(baseFileName, ecCtx); err != nil {
+ t.Fatalf("write ec files: %v", err)
+ }
+ if err := erasure_coding.WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil {
+ t.Fatalf("write .ecx: %v", err)
+ }
+ if err := os.WriteFile(baseFileName+".ecj", nil, 0o644); err != nil {
+ t.Fatalf("write .ecj: %v", err)
+ }
+ if err := volume_info.SaveVolumeInfo(baseFileName+".vif", &volume_server_pb.VolumeInfo{
+ Version: uint32(needle.GetCurrentVersion()),
+ DatFileSize: datSize.Size(),
+ EcShardConfig: &volume_server_pb.EcShardConfig{
+ DataShards: uint32(ecCtx.DataShards),
+ ParityShards: uint32(ecCtx.ParityShards),
+ BlockSize: ecCtx.BlockSize,
+ },
+ }); err != nil {
+ t.Fatalf("save .vif: %v", err)
+ }
+ for _, ext := range []string{".dat", ".idx"} {
+ if err := os.Remove(baseFileName + ext); err != nil {
+ t.Fatalf("remove %s: %v", ext, err)
+ }
+ }
+
+ for shardId := 0; shardId < erasure_coding.TotalShardsCount; shardId++ {
+ if err := store.MountEcShards("", vid, erasure_coding.ShardId(shardId), ""); err != nil {
+ t.Fatalf("mount shard %d: %v", shardId, err)
+ }
+ }
+ ecVolume, found := store.Locations[0].FindEcVolume(vid)
+ if !found {
+ t.Fatal("ec volume not mounted")
+ }
+ if ecVolume.ECContext.BlockSize != ecCtx.BlockSize {
+ t.Fatalf("mounted BlockSize = %d, want %d", ecVolume.ECContext.BlockSize, ecCtx.BlockSize)
+ }
+
+ // Every shard is local, so seed the location cache to keep the read off the
+ // master this test does not have.
+ ecVolume.ShardLocationsLock.Lock()
+ for shardId := 0; shardId < erasure_coding.TotalShardsCount; shardId++ {
+ ecVolume.ShardLocations[erasure_coding.ShardId(shardId)] = []pb.ServerAddress{"localhost:8080"}
+ }
+ ecVolume.ShardLocationsRefreshTime = time.Now()
+ ecVolume.ShardLocationsLock.Unlock()
+ return ecVolume
+}
+
+func randomNeedleOfSize(id uint64, size int) *needle.Needle {
+ n := new(needle.Needle)
+ n.Id = types.Uint64ToNeedleId(id)
+ n.Data = make([]byte, size)
+ rand.New(rand.NewSource(int64(id))).Read(n.Data)
+ n.Checksum = needle.NewCRC(n.Data)
+ return n
+}
+
// A needle larger than one EC block is split over consecutive blocks, which
// live on different shards. The intervals are read concurrently, so check they
// still come back in order.
func TestReadEcShardNeedleSpanningBlocks(t *testing.T) {
+ store := newTestStore(t, 1)
const vid = needle.VolumeId(7)
- store, ecVolume, n := newLocalEcVolume(t, vid)
+
+ n := randomNeedleOfSize(42, 3*erasure_coding.ErasureCodingSmallBlockSize+1234)
+ ecVolume := encodeAndMountEcVolume(t, store, vid, []*needle.Needle{n})
_, _, intervals, err := ecVolume.LocateEcShardNeedle(n.Id, ecVolume.Version)
if err != nil {
@@ -31,3 +126,52 @@ func TestReadEcShardNeedleSpanningBlocks(t *testing.T) {
t.Fatalf("read back %d bytes, want the %d written", len(got.Data), len(n.Data))
}
}
+
+// On a volume big enough for multi-MB blocks, the uniform layout keeps a
+// needle smaller than the block in one interval on one shard, and reads every
+// needle back intact.
+func TestReadEcShardNeedleUniformLayout(t *testing.T) {
+ store := newTestStore(t, 1)
+ const vid = needle.VolumeId(8)
+
+ // ~26MB of needles: block size becomes 3MB, so the layouts diverge and a
+ // 1MB needle would stripe across shards under the legacy layout.
+ var needles []*needle.Needle
+ for i := uint64(1); i <= 5; i++ {
+ needles = append(needles, randomNeedleOfSize(i, 4*1024*1024))
+ }
+ for i := uint64(6); i <= 11; i++ {
+ needles = append(needles, randomNeedleOfSize(i, 1024*1024))
+ }
+ ecVolume := encodeAndMountEcVolume(t, store, vid, needles)
+ if ecVolume.ECContext.BlockSize <= erasure_coding.ErasureCodingSmallBlockSize {
+ t.Fatalf("block size %d does not diverge from the legacy layout", ecVolume.ECContext.BlockSize)
+ }
+
+ // With 3MB blocks a 1MB needle sits inside one block (bar a boundary
+ // straddle), where the legacy layout would stripe it at 1MB granularity.
+ singleInterval := false
+ for _, n := range needles[5:] {
+ _, _, intervals, err := ecVolume.LocateEcShardNeedle(n.Id, ecVolume.Version)
+ if err != nil {
+ t.Fatalf("locate needle %v: %v", n.Id, err)
+ }
+ if len(intervals) == 1 {
+ singleInterval = true
+ }
+ }
+ if !singleInterval {
+ t.Fatal("no 1MB needle mapped to a single interval; uniform layout not in effect")
+ }
+
+ for _, n := range needles {
+ got := new(needle.Needle)
+ got.Id = n.Id
+ if _, err := store.ReadEcShardNeedle(vid, got, nil); err != nil {
+ t.Fatalf("read needle %v: %v", n.Id, err)
+ }
+ if !bytes.Equal(got.Data, n.Data) {
+ t.Fatalf("needle %v: read back %d bytes that do not match the %d written", n.Id, len(got.Data), len(n.Data))
+ }
+ }
+}
diff --git a/weed/storage/store_ec_recover_fanout_test.go b/weed/storage/store_ec_recover_fanout_test.go
index c30a3f443f..0bd75286e5 100644
--- a/weed/storage/store_ec_recover_fanout_test.go
+++ b/weed/storage/store_ec_recover_fanout_test.go
@@ -95,7 +95,8 @@ func writeEcVolumeFiles(t *testing.T, dir string, vid needle.VolumeId) (baseFile
if err != nil {
t.Fatalf("stat .dat: %v", err)
}
- if _, err := erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext()); err != nil {
+ ecCtx := erasure_coding.BackgroundECContext()
+ if _, err := erasure_coding.WriteEcFiles(baseFileName, ecCtx); err != nil {
t.Fatalf("write ec files: %v", err)
}
if err := erasure_coding.WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil {
@@ -110,6 +111,10 @@ func writeEcVolumeFiles(t *testing.T, dir string, vid needle.VolumeId) (baseFile
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: erasure_coding.DataShardsCount,
ParityShards: erasure_coding.ParityShardsCount,
+ // The encode writes the uniform layout; a .vif that omitted its
+ // block size would mount these shards as legacy and read every
+ // interval at the wrong offset.
+ BlockSize: ecCtx.BlockSize,
},
}); err != nil {
t.Fatalf("save .vif: %v", err)
@@ -162,7 +167,7 @@ func TestRecoverOneRemoteEcShardIntervalUsesLocalShards(t *testing.T) {
if err != nil {
t.Fatalf("locate needle: %v", err)
}
- shardIdToRecover, actualOffset := store.IntervalToShardIdAndOffset(intervals[0])
+ shardIdToRecover, actualOffset := ecVolume.IntervalToShardIdAndOffset(intervals[0])
got := make([]byte, intervals[0].Size)
nRead, _, err := store.recoverOneRemoteEcShardInterval(n.Id, ecVolume, shardIdToRecover, got, actualOffset)
diff --git a/weed/storage/store_ec_scrub.go b/weed/storage/store_ec_scrub.go
index 08c002682a..4a5c11169e 100644
--- a/weed/storage/store_ec_scrub.go
+++ b/weed/storage/store_ec_scrub.go
@@ -47,14 +47,14 @@ func (s *Store) ScrubEcVolume(vid needle.VolumeId, mode volume_server_pb.VolumeS
shardIds := make([]erasure_coding.ShardId, len(intervals))
for i, iv := range intervals {
- sid, _ := s.IntervalToShardIdAndOffset(iv)
+ sid, _ := ecv.IntervalToShardIdAndOffset(iv)
shardIds[i] = sid
}
slices.Sort(shardIds)
for i, iv := range intervals {
chunk := make([]byte, iv.Size)
- shardId, offset := s.IntervalToShardIdAndOffset(iv)
+ shardId, offset := ecv.IntervalToShardIdAndOffset(iv)
// try a local shard read first...
if err := s.readLocalEcShardInterval(ecv, shardId, chunk, offset); err == nil {
diff --git a/weed/storage/store_ec_scrub_reads_test.go b/weed/storage/store_ec_scrub_reads_test.go
index d8a1758b34..f30ff853a1 100644
--- a/weed/storage/store_ec_scrub_reads_test.go
+++ b/weed/storage/store_ec_scrub_reads_test.go
@@ -45,7 +45,8 @@ func newLocalEcVolume(t *testing.T, vid needle.VolumeId) (*Store, *erasure_codin
if err != nil {
t.Fatalf("stat .dat: %v", err)
}
- if _, err := erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext()); err != nil {
+ ecCtx := erasure_coding.BackgroundECContext()
+ if _, err := erasure_coding.WriteEcFiles(baseFileName, ecCtx); err != nil {
t.Fatalf("write ec files: %v", err)
}
if err := erasure_coding.WriteSortedFileFromIdx(baseFileName, ".ecx"); err != nil {
@@ -60,6 +61,10 @@ func newLocalEcVolume(t *testing.T, vid needle.VolumeId) (*Store, *erasure_codin
EcShardConfig: &volume_server_pb.EcShardConfig{
DataShards: erasure_coding.DataShardsCount,
ParityShards: erasure_coding.ParityShardsCount,
+ // The encode writes the uniform layout; a .vif that omitted its
+ // block size would mount these shards as legacy and read every
+ // interval at the wrong offset.
+ BlockSize: ecCtx.BlockSize,
},
}); err != nil {
t.Fatalf("save .vif: %v", err)
diff --git a/weed/worker/tasks/erasure_coding/ec_task.go b/weed/worker/tasks/erasure_coding/ec_task.go
index ac8cbe787c..b4ba0f126b 100644
--- a/weed/worker/tasks/erasure_coding/ec_task.go
+++ b/weed/worker/tasks/erasure_coding/ec_task.go
@@ -51,6 +51,11 @@ type ErasureCodingTask struct {
// pre-distribute sweep. deleteOriginalVolume skips these so it does not
// re-delete and remove the now-EC .vif those servers share.
emptyReplicasDeleted map[string]bool
+
+ // encodedBlockSize is the shard block layout WriteEcFiles actually encoded
+ // with, read back off the EC context. Every holder must report serving the
+ // same one before the source volume may be deleted.
+ encodedBlockSize int64
}
// NewErasureCodingTask creates a new unified EC task instance
@@ -583,16 +588,23 @@ func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string
}
// Generate EC shard files (.ec00 ~ .ec13)
- ecBitrot, err := erasure_coding.WriteEcFiles(baseName, erasure_coding.BackgroundECContext())
+ ecCtx := erasure_coding.BackgroundECContext()
+ ecBitrot, err := erasure_coding.WriteEcFiles(baseName, ecCtx)
if err != nil {
return nil, fmt.Errorf("failed to generate EC shard files: %w", err)
}
+ // The layout the shards were actually written in, for the holder agreement
+ // check before the source is deleted.
+ t.encodedBlockSize = ecCtx.BlockSize
// Persist the bitrot checksum sidecar (generation 0) alongside the shards so
- // it travels with them during distribution. Best-effort: a failed sidecar
- // write leaves the generation unprotected rather than failing the encode.
+ // it travels with them during distribution. Protection was asked for, and
+ // this write is orders of magnitude smaller than the shards that just
+ // landed: if it fails the disk is in trouble, and continuing would delete
+ // the source replicas in exchange for a generation that is both unprotected
+ // and missing the geometry record the .vif fallback reads.
if erasure_coding.BitrotProtectionEnabled && ecBitrot != nil {
if serr := erasure_coding.SaveBitrotSidecar(erasure_coding.BitrotSidecarPath(baseName, 0), ecBitrot); serr != nil {
- glog.Warningf("failed to write EC bitrot sidecar for %s: %v", baseName, serr)
+ return nil, fmt.Errorf("write EC bitrot sidecar for %s: %w", baseName, serr)
}
}
@@ -662,6 +674,7 @@ func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string
if ecBitrot != nil && ecBitrot.EcShardConfig != nil {
ecShardConfig.DataShards = ecBitrot.EcShardConfig.DataShards
ecShardConfig.ParityShards = ecBitrot.EcShardConfig.ParityShards
+ ecShardConfig.BlockSize = ecBitrot.EcShardConfig.BlockSize
}
volumeInfo := &volume_server_pb.VolumeInfo{
Version: uint32(needle.GetCurrentVersion()),
@@ -675,30 +688,40 @@ func (t *ErasureCodingTask) generateEcShardsLocally(localFiles map[string]string
} else {
glog.Warningf("stat %s for .vif dat file size: %v", datFile, err)
}
+ // The .vif carries the shard block layout the holders will read through, so
+ // neither the write nor the inclusion below is optional: an encode that
+ // distributed shards without it would leave every reader falling back to
+ // the legacy layout, and the task deletes the source replicas afterwards.
if err := volume_info.SaveVolumeInfo(vifFile, volumeInfo); err != nil {
- glog.Warningf("Failed to create .vif file: %v", err)
- } else {
- shardFiles["vif"] = vifFile
- if info, err := os.Stat(vifFile); err == nil {
- t.GetLogger().WithFields(map[string]interface{}{
- "file_type": "vif",
- "file_path": vifFile,
- "size_bytes": info.Size(),
- }).Info("Volume info file generated")
- }
+ return nil, fmt.Errorf("write %s: %w", vifFile, err)
}
+ vifInfo, err := os.Stat(vifFile)
+ if err != nil {
+ return nil, fmt.Errorf("stat %s for distribution: %w", vifFile, err)
+ }
+ shardFiles["vif"] = vifFile
+ t.GetLogger().WithFields(map[string]interface{}{
+ "file_type": "vif",
+ "file_path": vifFile,
+ "size_bytes": vifInfo.Size(),
+ }).Info("Volume info file generated")
// Add the generation-0 bitrot checksum sidecar so it is distributed with
// the shards (DistributeEcShards only ships files present in shardFiles).
- // Best-effort like the sidecar write above: if it is absent the holders
- // are simply unprotected rather than failing the encode.
+ // Strict when protection is enabled and the encoder produced a manifest —
+ // the write above already failed the encode otherwise — so the holders
+ // cannot end up with shards whose checksums stayed behind on the worker.
ecsumFile := erasure_coding.BitrotSidecarPath(baseName, 0)
- if info, err := os.Stat(ecsumFile); err == nil {
+ ecsumInfo, ecsumErr := os.Stat(ecsumFile)
+ if ecsumErr != nil && erasure_coding.BitrotProtectionEnabled && ecBitrot != nil {
+ return nil, fmt.Errorf("stat %s for distribution: %w", ecsumFile, ecsumErr)
+ }
+ if ecsumErr == nil {
shardFiles["ecsum"] = ecsumFile
t.GetLogger().WithFields(map[string]interface{}{
"file_type": "ecsum",
"file_path": ecsumFile,
- "size_bytes": info.Size(),
+ "size_bytes": ecsumInfo.Size(),
}).Info("EC bitrot checksum sidecar generated")
}
@@ -768,6 +791,20 @@ func (t *ErasureCodingTask) verifyEcShardsBeforeDelete(ctx context.Context) erro
"per_server": summary,
}).Warning("EC shard set incomplete but recoverable; proceeding with source deletion")
}
+
+ // Before anything irreversible: every holder that answered must report
+ // serving the layout these shards were encoded in. A holder too old to
+ // know the uniform layout mounts them as legacy and returns wrong bytes,
+ // and the source volume is the only remaining correct copy.
+ if err := erasure_coding.RequireAgreedBlockLayout(t.volumeID, t.encodedBlockSize, perServer); err != nil {
+ t.GetLogger().WithFields(map[string]interface{}{
+ "volume_id": t.volumeID,
+ "per_server": summary,
+ "error": err.Error(),
+ }).Error("EC holders disagree on the shard block layout — source volume will be kept")
+ return err
+ }
+
return nil
}