Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

fix(replication): verify-before-destroy in VolumeCopy, check.disk, and over-replication trim (#9943)

Chris Luc2591b439559
diff --git a/seaweed-volume/src/server/grpc_server.rs b/seaweed-volume/src/server/grpc_server.rs
index 2f20e58bba..02e0e808e5 100644
--- a/seaweed-volume/src/server/grpc_server.rs
+++ b/seaweed-volume/src/server/grpc_server.rs
@@ -1080,21 +1080,15 @@ impl VolumeServer for VolumeGrpcService {
let req = request.into_inner();
let vid = VolumeId(req.volume_id);
- // If volume already exists locally, delete it first
- {
+ // A pre-existing local replica is NOT deleted up front. Deleting before
+ // the source is confirmed reachable destroys a healthy copy on a
+ // transient source outage (and, on retry, can lose the volume
+ // entirely). The delete is deferred until read_volume_file_status below
+ // proves the source holds the volume; readability alone is the gate.
+ let had_existing_volume = {
let store = self.state.store.read().unwrap();
- if store.find_volume(vid).is_some() {
- drop(store);
- let mut store = self.state.store.write().unwrap();
- // keep remote data: the inbound copy carries a .vif that may
- // point at the same cloud-tier object the existing volume
- // references.
- store.delete_volume(vid, false, true).map_err(|e| {
- Status::internal(format!("failed to delete existing volume {}: {}", vid, e))
- })?;
- self.state.volume_state_notify.notify_one();
- }
- }
+ store.find_volume(vid).is_some()
+ };
// Parse source_data_node address: "ip:port.grpcPort" or "ip:port" (grpc = port + 10000)
let source = &req.source_data_node;
@@ -1135,6 +1129,19 @@ impl VolumeServer for VolumeGrpcService {
.map_err(|e| Status::internal(format!("read volume file status failed, {}", e)))?
.into_inner();
+ // Source is reachable and holds the volume: only now is it safe to drop
+ // an existing local replica before overwriting its files.
+ if had_existing_volume {
+ let mut store = self.state.store.write().unwrap();
+ // keep remote data: the inbound copy carries a .vif that may point
+ // at the same cloud-tier object the existing volume references.
+ store.delete_volume(vid, false, true).map_err(|e| {
+ Status::internal(format!("failed to delete existing volume {}: {}", vid, e))
+ })?;
+ drop(store);
+ self.state.volume_state_notify.notify_one();
+ }
+
let requested_disk_type = if !req.disk_type.is_empty() {
DiskType::from_string(&req.disk_type)
} else {
@@ -1166,9 +1173,12 @@ impl VolumeServer for VolumeGrpcService {
let idx_base_name =
crate::storage::volume::volume_file_name(&idx_base, &vol_info.collection, vid);
- // Write a .note file to indicate copy in progress
+ // Write a .note file to indicate copy in progress. A leftover note
+ // fails the volume load on restart, so a write failure must abort.
let note_path = format!("{}.note", data_base_name);
- let _ = std::fs::write(&note_path, format!("copying from {}", source));
+ std::fs::write(&note_path, format!("copying from {}", source)).map_err(|e| {
+ Status::internal(format!("write .note for volume {}: {}", vid, e))
+ })?;
let has_remote_dat = vol_info
.volume_info
@@ -1299,8 +1309,16 @@ impl VolumeServer for VolumeGrpcService {
let _ = set_file_mtime(&vif_path, vif_modified_ts_ns);
}
- // Remove the .note file
- let _ = std::fs::remove_file(&note_path);
+ // Remove the .note file. A leftover note fails the load on the
+ // next restart, so a removal failure must fail the copy.
+ if let Err(e) = std::fs::remove_file(&note_path) {
+ if e.kind() != std::io::ErrorKind::NotFound {
+ return Err(Status::internal(format!(
+ "remove .note for volume {}: {}",
+ vid, e
+ )));
+ }
+ }
// Verify file sizes
if !has_remote_dat {
diff --git a/seaweed-volume/src/storage/disk_location.rs b/seaweed-volume/src/storage/disk_location.rs
index d3c4a01798..ca5ea407cb 100644
--- a/seaweed-volume/src/storage/disk_location.rs
+++ b/seaweed-volume/src/storage/disk_location.rs
@@ -125,19 +125,6 @@ impl DiskLocation {
let volume_name = volume_file_name(&self.directory, &collection, vid);
let idx_name = volume_file_name(&self.idx_directory, &collection, vid);
- // Check for incomplete volume (.note file means a VolumeCopy was interrupted)
- let note_path = format!("{}.note", volume_name);
- if std::path::Path::new(&note_path).exists() {
- let note = fs::read_to_string(&note_path).unwrap_or_default();
- warn!(
- volume_id = vid.0,
- "volume was not completed: {}, removing files", note
- );
- remove_volume_files(&volume_name, false);
- remove_volume_files(&idx_name, false);
- continue;
- }
-
// Sweep a leftover empty `.dat` stub (a phantom from the pre-fix
// loader) before it loads as a phantom volume or blocks startup.
if remove_empty_ec_dat_stub(&volume_name, &idx_name, vid) {
@@ -181,6 +168,30 @@ impl DiskLocation {
let _ = fs::remove_file(&cpx_path);
}
+ // Check for an incomplete volume (.note means a VolumeCopy was
+ // interrupted). This runs BELOW the empty-stub sweep and EC
+ // validation: when an .ecx for this vid coexists on the disk, the
+ // regular and EC volumes share <base>.vif, so removing the
+ // incomplete regular copy must keep the .vif (keep_vif=true) or it
+ // would strip the EC volume's info file.
+ let note_path = format!("{}.note", volume_name);
+ if std::path::Path::new(&note_path).exists() {
+ let note = fs::read_to_string(&note_path).unwrap_or_default();
+ warn!(
+ volume_id = vid.0,
+ "volume was not completed: {}, removing files", note
+ );
+ // Re-check .ecx now (not the pre-validation ecx_exists): the
+ // invalid-EC cleanup above may have removed it, in which case
+ // the .vif is no longer shared and must not be preserved.
+ let keep_vif = std::path::Path::new(&ecx_path).exists()
+ || (self.idx_directory != self.directory
+ && std::path::Path::new(&format!("{}.ecx", volume_name)).exists());
+ remove_volume_files(&volume_name, keep_vif);
+ remove_volume_files(&idx_name, keep_vif);
+ continue;
+ }
+
// Skip if already loaded (e.g., from a previous call)
if self.volumes.contains_key(&vid) {
continue;
diff --git a/seaweed-volume/src/storage/store.rs b/seaweed-volume/src/storage/store.rs
index bb815be4cb..0ad57e2bf6 100644
--- a/seaweed-volume/src/storage/store.rs
+++ b/seaweed-volume/src/storage/store.rs
@@ -369,6 +369,16 @@ impl Store {
let vif_path = format!("{}.vif", base);
if std::path::Path::new(&dat_path).exists() || std::path::Path::new(&vif_path).exists()
{
+ // A persisting .note means the copy that produced these files
+ // never completed; mounting it would expose a truncated volume.
+ // Fail the mount so the caller (VolumeCopy) treats it as an error.
+ let note_path = format!("{}.note", base);
+ if std::path::Path::new(&note_path).exists() {
+ return Err(VolumeError::Io(io::Error::new(
+ io::ErrorKind::Other,
+ format!("volume {} copy incomplete: .note still present", vid),
+ )));
+ }
return loc.create_volume(
vid,
collection,
diff --git a/weed/server/volume_grpc_copy.go b/weed/server/volume_grpc_copy.go
index 89202d56d3..0d372982f3 100644
--- a/weed/server/volume_grpc_copy.go
+++ b/weed/server/volume_grpc_copy.go
@@ -31,20 +31,13 @@ func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stre
return err
}
- v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
- if v != nil {
-
- glog.V(0).Infof("volume %d already exists. deleted before copying...", req.VolumeId)
-
- // keep remote data: the inbound copy carries a .vif that may point at
- // the same cloud-tier object the existing volume references.
- err := vs.store.DeleteVolume(needle.VolumeId(req.VolumeId), false, true)
- if err != nil {
- return fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, err)
- }
-
- glog.V(0).Infof("deleted existing volume %d before copying.", req.VolumeId)
- }
+ // A pre-existing local replica is NOT deleted up front. Deleting before the
+ // source is confirmed reachable destroys a healthy copy on a transient
+ // source outage (and, on retry, can lose the volume entirely). The delete is
+ // deferred until ReadVolumeFileStatus below proves the source holds the
+ // volume; readability alone is the gate (size/count comparisons invert after
+ // divergent vacuum/compaction and would block valid re-replication).
+ hasExistingVolume := vs.store.GetVolume(needle.VolumeId(req.VolumeId)) != nil
// the master will not start compaction for read-only volumes, so it is safe to just copy files directly
// copy .dat and .idx files
@@ -65,6 +58,18 @@ func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stre
return fmt.Errorf("read volume file status failed, %w", err)
}
+ // Source is reachable and holds the volume: only now is it safe to drop
+ // an existing local replica before overwriting its files.
+ if hasExistingVolume {
+ glog.V(0).Infof("volume %d already exists. deleting before copying from %s...", req.VolumeId, req.SourceDataNode)
+ // keep remote data: the inbound copy carries a .vif that may point at
+ // the same cloud-tier object the existing volume references.
+ if delErr := vs.store.DeleteVolume(needle.VolumeId(req.VolumeId), false, true); delErr != nil {
+ return fmt.Errorf("failed to delete existing volume %d: %v", req.VolumeId, delErr)
+ }
+ glog.V(0).Infof("deleted existing volume %d before copying.", req.VolumeId)
+ }
+
diskType := volFileInfoResp.DiskType
if req.DiskType != "" {
diskType = req.DiskType
@@ -81,7 +86,12 @@ func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stre
indexBaseFileName = storage.VolumeFileName(location.IdxDirectory, volFileInfoResp.Collection, int(req.VolumeId))
hasRemoteDatFile = volFileInfoResp.VolumeInfo != nil && len(volFileInfoResp.VolumeInfo.Files) > 0
- util.WriteFile(dataBaseFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755)
+ // The .note marks the copy as in-progress; a leftover note fails the
+ // volume load on restart, so a write failure must abort the copy.
+ if noteErr := util.WriteFile(dataBaseFileName+".note", []byte(fmt.Sprintf("copying from %s", req.SourceDataNode)), 0755); noteErr != nil {
+ err = noteErr
+ return fmt.Errorf("write .note for volume %d: %w", req.VolumeId, noteErr)
+ }
defer func() {
if err != nil {
@@ -163,7 +173,12 @@ func (vs *VolumeServer) VolumeCopy(req *volume_server_pb.VolumeCopyRequest, stre
os.Chtimes(dataBaseFileName+".vif", time.Unix(0, modifiedTsNs), time.Unix(0, modifiedTsNs))
}
- os.Remove(dataBaseFileName + ".note")
+ // A leftover .note fails the load on the next restart, so a removal
+ // failure must fail the copy rather than be silently swallowed.
+ if noteErr := os.Remove(dataBaseFileName + ".note"); noteErr != nil && !os.IsNotExist(noteErr) {
+ err = noteErr
+ return fmt.Errorf("remove .note for volume %d: %w", req.VolumeId, noteErr)
+ }
return nil
})
diff --git a/weed/server/volume_grpc_copy_verify_test.go b/weed/server/volume_grpc_copy_verify_test.go
new file mode 100644
index 0000000000..ef10832abc
--- /dev/null
+++ b/weed/server/volume_grpc_copy_verify_test.go
@@ -0,0 +1,75 @@
+package weed_server
+
+import (
+ "context"
+ "testing"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
+ "google.golang.org/grpc/metadata"
+
+ "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+ "github.com/seaweedfs/seaweedfs/weed/stats"
+ "github.com/seaweedfs/seaweedfs/weed/storage"
+ "github.com/seaweedfs/seaweedfs/weed/storage/needle"
+ "github.com/seaweedfs/seaweedfs/weed/storage/types"
+ "github.com/seaweedfs/seaweedfs/weed/util"
+)
+
+// fakeVolumeCopyStream is a no-op VolumeServer_VolumeCopyServer; VolumeCopy
+// errors out before sending anything in this test.
+type fakeVolumeCopyStream struct {
+ grpc.ServerStream
+}
+
+func (s *fakeVolumeCopyStream) Send(*volume_server_pb.VolumeCopyResponse) error { return nil }
+func (s *fakeVolumeCopyStream) Context() context.Context { return context.Background() }
+func (s *fakeVolumeCopyStream) SetHeader(metadata.MD) error { return nil }
+func (s *fakeVolumeCopyStream) SendHeader(metadata.MD) error { return nil }
+func (s *fakeVolumeCopyStream) SetTrailer(metadata.MD) {}
+func (s *fakeVolumeCopyStream) SendMsg(any) error { return nil }
+func (s *fakeVolumeCopyStream) RecvMsg(any) error { return nil }
+
+// TestVolumeCopy_KeepsExistingReplicaWhenSourceUnreachable verifies the
+// verify-before-destroy invariant: a pre-existing healthy local replica must
+// NOT be deleted when the source cannot be reached. The pre-fix code deleted
+// the destination up front (and, on retry, could lose the volume entirely);
+// the fix defers the delete until the source ReadVolumeFileStatus succeeds.
+func TestVolumeCopy_KeepsExistingReplicaWhenSourceUnreachable(t *testing.T) {
+ dir := t.TempDir()
+ store := storage.NewStore(
+ grpc.WithTransportCredentials(insecure.NewCredentials()),
+ "127.0.0.1", 0, 0, "", "test-store",
+ []string{dir}, []int32{10}, []util.MinFreeSpace{{}},
+ dir, storage.NeedleMapInMemory,
+ []types.DiskType{types.HardDriveType}, [][]string{nil},
+ 0, stats.DiskIOProbeConfig{},
+ )
+
+ const vid = needle.VolumeId(42)
+ if err := store.AddVolume(vid, "", storage.NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0); err != nil {
+ t.Fatalf("AddVolume: %v", err)
+ }
+ if store.GetVolume(vid) == nil {
+ t.Fatalf("setup: volume %d should exist", vid)
+ }
+
+ vs := &VolumeServer{
+ store: store,
+ grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+ }
+
+ // 127.0.0.1:1 is unreachable, so ReadVolumeFileStatus on the source fails.
+ req := &volume_server_pb.VolumeCopyRequest{
+ VolumeId: uint32(vid),
+ SourceDataNode: "127.0.0.1:1",
+ }
+ err := vs.VolumeCopy(req, &fakeVolumeCopyStream{})
+ if err == nil {
+ t.Fatalf("VolumeCopy should fail when the source is unreachable")
+ }
+
+ if store.GetVolume(vid) == nil {
+ t.Fatalf("existing replica %d was destroyed before the source was verified", vid)
+ }
+}
diff --git a/weed/shell/command_volume_check_disk.go b/weed/shell/command_volume_check_disk.go
index af5b806a31..10e22588bf 100644
--- a/weed/shell/command_volume_check_disk.go
+++ b/weed/shell/command_volume_check_disk.go
@@ -22,7 +22,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/server/constants"
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
- "github.com/seaweedfs/seaweedfs/weed/storage/volume_replica"
"google.golang.org/grpc"
)
@@ -44,6 +43,12 @@ type volumeCheckDisk struct {
syncDeletions bool
fixReadOnly bool
nonRepairThreshold float64
+ // resurrectMissingNeedles controls whether a needle present on the source
+ // but entirely absent on the target is pushed back. Default false: an
+ // absent needle is indistinguishable from a vacuumed delete, so the safe
+ // default never raises deleted data. No caller sets this true today; it is
+ // the seam for a future tombstone-aware repair path.
+ resurrectMissingNeedles bool
ewg *ErrorWaitGroup
}
@@ -511,9 +516,8 @@ func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.Me
// hash join, can be more efficient
var missingNeedles []needle_map.NeedleValue
var partiallyDeletedNeedles []needle_map.NeedleValue
+ var skippedAbsentNeedles int
var counter int
- doCutoffOfLastNeedle := true
- cutoffFromAtNs := uint64(vcd.now.UnixNano())
minuend.DescendingVisit(func(minuendValue needle_map.NeedleValue) error {
counter++
@@ -521,27 +525,34 @@ func (vcd *volumeCheckDisk) doVolumeCheckDisk(minuend, subtrahend *needle_map.Me
if minuendValue.Size.IsDeleted() {
return nil
}
- if doCutoffOfLastNeedle {
- if needleMeta, err := volume_replica.ReadNeedleMeta(vcd.grpcDialOption(), pb.NewServerAddressFromDataNode(source.location.dataNode), source.info.Id, minuendValue); err == nil {
- // needles older than the cutoff time are not missing yet
- if needleMeta.AppendAtNs > cutoffFromAtNs {
- return nil
- }
- doCutoffOfLastNeedle = false
- }
+ // A key present-and-live on the source but entirely absent on the
+ // target is ambiguous: either a genuine missing write, or a needle
+ // that was deleted on the target and then vacuumed away (its index
+ // entry, including any tombstone, is gone after vacuum). An
+ // individual needle's AppendAtNs has no monotonic relation to a
+ // vacuum watermark, so it cannot distinguish the two. Without
+ // positive proof the absence is a missing write (rather than a
+ // vacuumed delete), the safe default is to NOT resurrect: a real
+ // missing write may go unrepaired until a tombstone-aware path
+ // exists, but we never raise back data the operator deleted.
+ if !vcd.resurrectMissingNeedles {
+ skippedAbsentNeedles++
+ return nil
}
missingNeedles = append(missingNeedles, minuendValue)
} else {
if minuendValue.Size.IsDeleted() && !subtrahendValue.Size.IsDeleted() {
partiallyDeletedNeedles = append(partiallyDeletedNeedles, minuendValue)
}
- if doCutoffOfLastNeedle {
- doCutoffOfLastNeedle = false
- }
}
return nil
})
+ if skippedAbsentNeedles > 0 {
+ vcd.write("volume %d %s: not resurrecting %d needle(s) absent on %s (cannot prove they are missing writes vs vacuumed deletes)",
+ source.info.Id, source.location.dataNode.Id, skippedAbsentNeedles, target.location.dataNode.Id)
+ }
+
vcd.write("volume %d %s has %d entries, %s missed %d and partially deleted %d entries",
source.info.Id, source.location.dataNode.Id, counter, target.location.dataNode.Id, len(missingNeedles), len(partiallyDeletedNeedles))
diff --git a/weed/shell/command_volume_check_disk_test.go b/weed/shell/command_volume_check_disk_test.go
index ec958fbc4f..8a31e5cdfc 100644
--- a/weed/shell/command_volume_check_disk_test.go
+++ b/weed/shell/command_volume_check_disk_test.go
@@ -6,8 +6,66 @@ import (
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+ "github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
+ "github.com/seaweedfs/seaweedfs/weed/storage/types"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials/insecure"
)
+// TestDoVolumeCheckDiskDoesNotResurrectAbsentNeedle verifies that a needle
+// present-and-live on the source but entirely absent on the target is NOT
+// pushed back by default. Such an absence is indistinguishable from a needle
+// that was deleted on the target and then vacuumed away, so resurrecting it
+// would raise back data the operator deleted.
+func TestDoVolumeCheckDiskDoesNotResurrectAbsentNeedle(t *testing.T) {
+ sourceDB, targetDB := needle_map.NewMemDb(), needle_map.NewMemDb()
+ defer sourceDB.Close()
+ defer targetDB.Close()
+
+ // Source has a live needle; target lacks it (e.g. deleted+vacuumed there).
+ if err := sourceDB.Set(types.NeedleId(1001), types.ToOffset(8), types.Size(123)); err != nil {
+ t.Fatalf("seed source: %v", err)
+ }
+
+ var buf bytes.Buffer
+ vcd := &volumeCheckDisk{
+ commandEnv: &CommandEnv{
+ option: &ShellOptions{
+ GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
+ },
+ },
+ writer: &buf,
+ now: time.Now(),
+ applyChanges: false, // simulation: should not even read the source blob
+ nonRepairThreshold: 1,
+ // resurrectMissingNeedles left false: the safe default.
+ }
+
+ // Source points at an unreachable address: with the fix no blob read is
+ // attempted, so this is never contacted. The pre-fix code queues the
+ // absent needle and tries to read it from here, surfacing as an error.
+ source := &VolumeReplica{
+ location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "127.0.0.1:1"}},
+ info: &master_pb.VolumeInformationMessage{Id: 7},
+ }
+ target := &VolumeReplica{
+ location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "127.0.0.1:2"}},
+ info: &master_pb.VolumeInformationMessage{Id: 7},
+ }
+
+ // With the safe default, the absent needle is skipped before any network
+ // read, so this returns cleanly with no changes. On the pre-fix code the
+ // needle is queued and a source blob read is attempted, which has no server
+ // to reach and surfaces as an error (or a resurrection) instead.
+ hasChanges, err := vcd.doVolumeCheckDisk(sourceDB, targetDB, source, target)
+ if err != nil {
+ t.Fatalf("doVolumeCheckDisk returned error: %v", err)
+ }
+ if hasChanges {
+ t.Fatalf("absent-on-target needle was resurrected; expected no changes")
+ }
+}
+
type testCommandVolumeCheckDisk struct {
commandVolumeCheckDisk
}
diff --git a/weed/shell/command_volume_fix_replication.go b/weed/shell/command_volume_fix_replication.go
index e4af93d289..73497e5304 100644
--- a/weed/shell/command_volume_fix_replication.go
+++ b/weed/shell/command_volume_fix_replication.go
@@ -219,7 +219,12 @@ func collectVolumeReplicaLocations(topologyInfo *master_pb.TopologyInfo) (map[ui
type SelectOneVolumeFunc func(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica
-func checkOneVolume(a *VolumeReplica, b *VolumeReplica, writer io.Writer, commandEnv *CommandEnv) (err error) {
+// checkOneVolume compares the index of replica a against b. With
+// applyChanges=false it is a read-only divergence check; the over-replication
+// trim must use that mode so it does not push the soon-to-be-deleted replica's
+// needles into the survivor (which would resurrect data and is the opposite of
+// a safe trim).
+func checkOneVolume(a *VolumeReplica, b *VolumeReplica, writer io.Writer, commandEnv *CommandEnv, applyChanges bool) (err error) {
aDB, bDB := needle_map.NewMemDb(), needle_map.NewMemDb()
defer func() {
aDB.Close()
@@ -232,7 +237,7 @@ func checkOneVolume(a *VolumeReplica, b *VolumeReplica, writer io.Writer, comman
now: time.Now(),
verbose: false,
- applyChanges: true,
+ applyChanges: applyChanges,
syncDeletions: false,
nonRepairThreshold: float64(1),
}
@@ -261,6 +266,10 @@ func (c *commandVolumeFixReplication) deleteOneVolume(commandEnv *CommandEnv, wr
replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replicas[0].info.ReplicaPlacement))
replica := selectOneVolumeFn(replicas, replicaPlacement)
+ if replica == nil {
+ fmt.Fprintf(writer, "skip trimming volume %d: no safe replica to delete (would leave only read-only survivors)\n", vid)
+ continue
+ }
// check collection name pattern
if *c.collectionPattern != "" {
@@ -302,7 +311,9 @@ func (c *commandVolumeFixReplication) deleteOneVolume(commandEnv *CommandEnv, wr
if replicaB.location.dataNode == replica.location.dataNode {
continue
}
- if checkErr = checkOneVolume(replica, replicaB, writer, commandEnv); checkErr != nil {
+ // Read-only divergence check only: never write the doomed
+ // replica's needles into a survivor while trimming.
+ if checkErr = checkOneVolume(replica, replicaB, writer, commandEnv, false); checkErr != nil {
fmt.Fprintf(writer, "sync volume %d on %s and %s: %v\n", replica.info.Id, replica.location.dataNode.Id, replicaB.location.dataNode.Id, checkErr)
break
}
@@ -625,8 +636,20 @@ func countReplicas(replicas []*VolumeReplica) (diffDc, diffRack, diffNode map[st
return
}
-func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica {
- slices.SortFunc(replicas, func(a, b *VolumeReplica) int {
+// pickOneReplicaToDelete selects the replica to trim when over-replicated.
+// It only ever removes the smallest of multiple healthy writable replicas: a
+// ReadOnly/integrity-flagged replica is never chosen for deletion, and the
+// trim is refused (returns nil) when removing a writable replica would leave
+// only ReadOnly survivors. VolumeStatus file_count>0 alone cannot prove the
+// survivors' .dat is readable, so we do not over-claim survivor health.
+// pickSmallestReplica returns the smallest replica (ties broken by oldest then
+// lowest compact revision), or nil for an empty set.
+func pickSmallestReplica(replicas []*VolumeReplica) *VolumeReplica {
+ if len(replicas) == 0 {
+ return nil
+ }
+ sorted := slices.Clone(replicas)
+ slices.SortFunc(sorted, func(a, b *VolumeReplica) int {
if a.info.Size != b.info.Size {
return int(a.info.Size - b.info.Size)
}
@@ -638,9 +661,39 @@ func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_b
}
return 0
})
+ return sorted[0]
+}
- return replicas[0]
-
+func pickOneReplicaToDelete(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) *VolumeReplica {
+ // Over-replication trim: only ever remove a writable replica, and only
+ // when another writable one survives, so a healthy copy is never deleted
+ // down to a read-only (e.g. full or integrity-flagged) survivor.
+ var writable []*VolumeReplica
+ for _, r := range replicas {
+ if !r.info.ReadOnly {
+ writable = append(writable, r)
+ }
+ }
+ if len(writable) < 2 {
+ return nil
+ }
+ // Prefer a writable replica whose removal still satisfies placement, so the
+ // trim does not strip the only replica in a required failure domain. Fall
+ // back to the smallest writable if none keeps placement (a later misplaced
+ // cycle then re-balances).
+ var placementSafe []*VolumeReplica
+ for i, r := range replicas {
+ if r.info.ReadOnly {
+ continue
+ }
+ if !isMisplaced(otherThan(replicas, i), replicaPlacement) {
+ placementSafe = append(placementSafe, r)
+ }
+ }
+ if len(placementSafe) > 0 {
+ return pickSmallestReplica(placementSafe)
+ }
+ return pickSmallestReplica(writable)
}
// check and fix misplaced volumes
@@ -669,6 +722,10 @@ func otherThan(replicas []*VolumeReplica, index int) (others []*VolumeReplica) {
func pickOneMisplacedVolume(replicas []*VolumeReplica, replicaPlacement *super_block.ReplicaPlacement) (toDelete *VolumeReplica) {
+ // Relocation, not over-replication: pick the smallest replica to delete
+ // and recreate at a correct placement. Unlike the trim this must still act
+ // on read-only replicas (e.g. a full but misplaced volume), so it does not
+ // use pickOneReplicaToDelete's writable-survivor guard.
var deletionCandidates []*VolumeReplica
for i := 0; i < len(replicas); i++ {
others := otherThan(replicas, i)
@@ -676,10 +733,10 @@ func pickOneMisplacedVolume(replicas []*VolumeReplica, replicaPlacement *super_b
deletionCandidates = append(deletionCandidates, replicas[i])
}
}
- if len(deletionCandidates) > 0 {
- return pickOneReplicaToDelete(deletionCandidates, replicaPlacement)
+ if toDelete = pickSmallestReplica(deletionCandidates); toDelete != nil {
+ return toDelete
}
- return pickOneReplicaToDelete(replicas, replicaPlacement)
+ return pickSmallestReplica(replicas)
}
diff --git a/weed/shell/command_volume_fix_replication_test.go b/weed/shell/command_volume_fix_replication_test.go
index 5f2318c327..46d3c008b3 100644
--- a/weed/shell/command_volume_fix_replication_test.go
+++ b/weed/shell/command_volume_fix_replication_test.go
@@ -439,6 +439,127 @@ func TestPickingMisplacedVolumeToDelete(t *testing.T) {
}
+func TestPickOneReplicaToDeleteSkipsReadOnlySurvivor(t *testing.T) {
+ replicaPlacement, _ := super_block.NewReplicaPlacementFromString("001")
+
+ // One writable replica and one read-only replica. The trim must never
+ // delete the only writable replica while the survivor is read-only:
+ // VolumeStatus alone cannot prove the read-only .dat is readable, so the
+ // trim is refused.
+ t.Run("refuses to delete the only writable replica while a read-only survivor remains", func(t *testing.T) {
+ replicas := []*VolumeReplica{
+ {
+ location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn-writable"}},
+ info: &master_pb.VolumeInformationMessage{Size: 90, ReadOnly: false},
+ },
+ {
+ location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn-readonly"}},
+ info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: true},
+ },
+ }
+ if got := pickOneReplicaToDelete(replicas, replicaPlacement); got != nil {
+ t.Fatalf("expected no replica to delete, got %s", got.location.dataNode.Id)
+ }
+ })
+
+ // All survivors read-only: never trim.
+ t.Run("refuses when all replicas are read-only", func(t *testing.T) {
+ replicas := []*VolumeReplica{
+ {
+ location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
+ info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: true},
+ },
+ {
+ location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
+ info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: true},
+ },
+ }
+ if got := pickOneReplicaToDelete(replicas, replicaPlacement); got != nil {
+ t.Fatalf("expected no replica to delete, got %s", got.location.dataNode.Id)
+ }
+ })
+
+ // Two healthy writable replicas plus a read-only one: trim the smallest
+ // writable, never the read-only one.
+ t.Run("trims the smallest writable replica and never the read-only one", func(t *testing.T) {
+ replicas := []*VolumeReplica{
+ {
+ location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn-big"}},
+ info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: false},
+ },
+ {
+ location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn-small"}},
+ info: &master_pb.VolumeInformationMessage{Size: 90, ReadOnly: false},
+ },
+ {
+ location: &location{"dc1", "r3", &master_pb.DataNodeInfo{Id: "dn-readonly"}},
+ info: &master_pb.VolumeInformationMessage{Size: 80, ReadOnly: true},
+ },
+ }
+ got := pickOneReplicaToDelete(replicas, replicaPlacement)
+ if got == nil {
+ t.Fatalf("expected a writable replica to delete, got nil")
+ }
+ if got.location.dataNode.Id != "dn-small" {
+ t.Fatalf("expected to delete smallest writable dn-small, got %s", got.location.dataNode.Id)
+ }
+ })
+}
+
+// The over-replication writable-survivor guard must NOT leak into the misplaced
+// relocation path: a misplaced volume whose replicas are all read-only (e.g. a
+// full volume) must still be relocated, picking the smallest replica to delete
+// and recreate at a correct placement.
+func TestPickOneMisplacedVolumeRelocatesReadOnlyReplicas(t *testing.T) {
+ replicaPlacement, _ := super_block.NewReplicaPlacementFromString("001")
+ replicas := []*VolumeReplica{
+ {
+ location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dn1"}},
+ info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: true},
+ },
+ {
+ location: &location{"dc1", "r2", &master_pb.DataNodeInfo{Id: "dn2"}},
+ info: &master_pb.VolumeInformationMessage{Size: 99, ReadOnly: true},
+ },
+ }
+ got := pickOneMisplacedVolume(replicas, replicaPlacement)
+ if got == nil {
+ t.Fatal("misplaced read-only volume must still be relocated, got nil")
+ }
+ if got.location.dataNode.Id != "dn2" {
+ t.Fatalf("expected to relocate smallest replica dn2, got %s", got.location.dataNode.Id)
+ }
+}
+
+// The over-replication trim must not strip the only replica in a required
+// failure domain: with "100" and replicas in dc1 + two in dc2, deleting the
+// smallest (dc1) leaves both survivors in dc2 and violates placement, so the
+// trim must instead delete a dc2 writable replica.
+func TestPickOneReplicaToDeletePreservesPlacement(t *testing.T) {
+ replicaPlacement, _ := super_block.NewReplicaPlacementFromString("100")
+ replicas := []*VolumeReplica{
+ {
+ location: &location{"dc1", "r1", &master_pb.DataNodeInfo{Id: "dc1-writable"}},
+ info: &master_pb.VolumeInformationMessage{Size: 90, ReadOnly: false},
+ },
+ {
+ location: &location{"dc2", "r2", &master_pb.DataNodeInfo{Id: "dc2-readonly"}},
+ info: &master_pb.VolumeInformationMessage{Size: 80, ReadOnly: true},
+ },
+ {
+ location: &location{"dc2", "r3", &master_pb.DataNodeInfo{Id: "dc2-writable"}},
+ info: &master_pb.VolumeInformationMessage{Size: 100, ReadOnly: false},
+ },
+ }
+ got := pickOneReplicaToDelete(replicas, replicaPlacement)
+ if got == nil {
+ t.Fatal("expected a replica to delete")
+ }
+ if got.location.dataNode.Id != "dc2-writable" {
+ t.Fatalf("expected to delete dc2-writable to preserve placement, got %s", got.location.dataNode.Id)
+ }
+}
+
func TestSatisfyReplicaCurrentLocation(t *testing.T) {
var tests = []testcase{
diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go
index 33312eb49a..38d41d60a1 100644
--- a/weed/storage/disk_location.go
+++ b/weed/storage/disk_location.go
@@ -250,8 +250,12 @@ func (l *DiskLocation) loadExistingVolume(dirEntry os.DirEntry, needleMapKind Ne
if util.FileExists(noteFile) {
note, _ := os.ReadFile(noteFile)
glog.Warningf("volume %s was not completed: %s", volumeName, string(note))
- removeVolumeFiles(l.Directory+"/"+volumeName, false)
- removeVolumeFiles(l.IdxDirectory+"/"+volumeName, false)
+ // Keep the .vif when an .ecx for this vid coexists on the disk: the
+ // regular and EC volumes share <base>.vif, so removing the incomplete
+ // regular copy must not strip the EC volume's info file.
+ keepVif := l.hasEcxFile(volumeName)
+ removeVolumeFiles(l.Directory+"/"+volumeName, keepVif)
+ removeVolumeFiles(l.IdxDirectory+"/"+volumeName, keepVif)
return false
}