Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

Fix scrubbing of deleted needles on EC volumes. (#10130)

Lisandro Pincac83bb4a80c
diff --git a/weed/server/volume_grpc_scrub.go b/weed/server/volume_grpc_scrub.go
index c306944ae5..3b967e35c6 100644
--- a/weed/server/volume_grpc_scrub.go
+++ b/weed/server/volume_grpc_scrub.go
@@ -124,7 +124,8 @@ func (vs *VolumeServer) ScrubEcVolume(ctx context.Context, req *volume_server_pb
case volume_server_pb.VolumeScrubMode_LOCAL:
files, shardInfos, serrs = v.ScrubLocal()
case volume_server_pb.VolumeScrubMode_FULL:
- files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId)
+ // TODO: expose force_deleted_needles_check in the RPC and weed shell.
+ files, shardInfos, serrs = vs.store.ScrubEcVolume(v.VolumeId, false)
case volume_server_pb.VolumeScrubMode_CHECKSUM:
// Verify each local shard's raw bytes against the bitrot sidecar,
// exercising cold parity shards. Read-only. ChecksumScrub's first
diff --git a/weed/storage/store_ec_scrub.go b/weed/storage/store_ec_scrub.go
index fbd9eccceb..2de3da1c4a 100644
--- a/weed/storage/store_ec_scrub.go
+++ b/weed/storage/store_ec_scrub.go
@@ -1,6 +1,7 @@
package storage
import (
+ "errors"
"fmt"
"slices"
@@ -12,7 +13,7 @@ import (
// ScrubEcVolume checks the full integrity of a EC volume, across both local and remote shards.
// Returns a count of processed file entries, slice of found broken shards, and slice of found errors.
-func (s *Store) ScrubEcVolume(vid needle.VolumeId) (int64, []*volume_server_pb.EcShardInfo, []error) {
+func (s *Store) ScrubEcVolume(vid needle.VolumeId, forceDeletedNeedlesCheck bool) (int64, []*volume_server_pb.EcShardInfo, []error) {
ecv, found := s.FindEcVolume(vid)
if !found {
return 0, nil, []error{fmt.Errorf("EC volume id %d not found", vid)}
@@ -78,7 +79,12 @@ func (s *Store) ScrubEcVolume(vid needle.VolumeId) (int64, []*volume_server_pb.E
n := needle.Needle{}
if err := n.ReadBytes(data, 0, size, ecv.Version); err != nil {
- errs = append(errs, fmt.Errorf("needle %d on EC volume %d: %v", id, ecv.VolumeId, err))
+ // needles flagged as deleted in the index but not in the volume (or vice-versa) cannot
+ // be properly hydrated, as the header read by needle.ReadBytes() will mismatch.
+ deleteSizeMismatch := size.IsDeleted() != (n.Size == 0)
+ if !errors.Is(err, needle.ErrorSizeMismatch) || !deleteSizeMismatch || forceDeletedNeedlesCheck {
+ errs = append(errs, fmt.Errorf("needle %d on EC volume %d: %v", id, ecv.VolumeId, err))
+ }
}
return nil