Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

feat(filer.sync.verify): reclassify chunk-slice-order ETag diffs as CHUNK_REORDER (#10177)

Jaehoon Kimebeab4b6ecf6
diff --git a/weed/command/filer_sync_verify.go b/weed/command/filer_sync_verify.go
index 0031eb0e36..378103f8e9 100644
--- a/weed/command/filer_sync_verify.go
+++ b/weed/command/filer_sync_verify.go
@@ -58,6 +58,12 @@ var cmdFilerSyncVerify = &Command{
-modifiedTimeAgo to skip recently-modified files (sync-lag tolerance) and
-isActivePassive for unidirectional comparison.
+ A chunk-derived ETag (no stored attr.Md5) can differ between clusters only
+ because the chunk slice was assembled in a different order, since ETagChunks
+ does not sort by offset. Such files are byte-identical and reported as
+ CHUNK_REORDER: not counted as errors, always counted in the summary, listed
+ only at -v=1.
+
Exits with code 0 on agreement, 2 on differences or operational errors.
`,
@@ -123,6 +129,7 @@ type VerifyResult struct {
missingCount atomic.Int64
sizeMismatch atomic.Int64
etagMismatch atomic.Int64
+ chunkReorder atomic.Int64 // content-equal; chunks differ only in slice order
onlyInB atomic.Int64
skippedRecent atomic.Int64
@@ -139,6 +146,7 @@ const (
diffOnlyInB // in B but not in A
diffSizeMismatch // size differs
diffETagMismatch // etag differs
+ diffChunkReorder // etag differs but content is equal (chunk slice order only)
)
// diffRecord is the JSON Lines schema for a single diff entry.
@@ -167,6 +175,7 @@ type summaryRecord struct {
Missing int64 `json:"missing"`
SizeMismatch int64 `json:"sizeMismatch"`
ETagMismatch int64 `json:"etagMismatch"`
+ ChunkReorder int64 `json:"chunkReorder"`
OnlyInB int64 `json:"onlyInB"`
TotalErrors int64 `json:"totalErrors"`
}
@@ -236,6 +245,7 @@ func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string,
Missing: result.missingCount.Load(),
SizeMismatch: result.sizeMismatch.Load(),
ETagMismatch: result.etagMismatch.Load(),
+ ChunkReorder: result.chunkReorder.Load(),
OnlyInB: result.onlyInB.Load(),
TotalErrors: totalErrors,
}
@@ -250,6 +260,9 @@ func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string,
fmt.Fprintf(os.Stdout, " Missing in B: %d\n", result.missingCount.Load())
fmt.Fprintf(os.Stdout, " Size mismatch: %d\n", result.sizeMismatch.Load())
fmt.Fprintf(os.Stdout, " ETag mismatch: %d\n", result.etagMismatch.Load())
+ if n := result.chunkReorder.Load(); n > 0 {
+ fmt.Fprintf(os.Stdout, " Chunk reorder (equal): %d (content-equal, not an error; use -v=1 to list)\n", n)
+ }
if !isActivePassive {
fmt.Fprintf(os.Stdout, " Only in B: %d\n", result.onlyInB.Load())
}
@@ -496,11 +509,88 @@ func compareEntries(dir string, entryA, entryB *filer_pb.Entry, result *VerifyRe
etagA := filer.ETag(entryA)
etagB := filer.ETag(entryB)
if etagA != etagB {
- reportDiff(diffETagMismatch, dir, entryA, entryB, result)
+ if isChunkReorder(entryA, entryB) {
+ // Same bytes, just a different chunk slice order — not a real diff.
+ reportDiff(diffChunkReorder, dir, entryA, entryB, result)
+ } else {
+ reportDiff(diffETagMismatch, dir, entryA, entryB, result)
+ }
return
}
}
+// isChunkReorder reports whether two entries with differing file ETags are
+// byte-identical, differing only in chunk slice order. filer.ETagChunks hashes
+// per-chunk MD5s in stored order without sorting by offset, so the same bytes
+// written by different paths (S3 multipart vs filer.backup) can hash
+// differently. If both chunk lists sorted by offset match element-wise on
+// (offset, size, ETag), the files cover the same bytes with the same content
+// and only the slice order differed.
+//
+// Applies only when both ETags come from chunks (a stored attr.Md5 is
+// order-independent, so a mismatch there is a real content difference) and the
+// chunks form a manifest-free, non-overlapping set whose bytes are fully
+// determined by the list — overlaps would need timestamp-based visible-interval
+// resolution, so those stay ETAG_MISMATCH.
+func isChunkReorder(entryA, entryB *filer_pb.Entry) bool {
+ if !usesChunkETag(entryA) || !usesChunkETag(entryB) {
+ return false
+ }
+ a := sortedSimpleChunks(entryA.GetChunks())
+ b := sortedSimpleChunks(entryB.GetChunks())
+ // nil means overlapping or manifest chunks we do not fast-path; a single
+ // chunk hashes directly and a differing count is a real divergence.
+ if a == nil || b == nil || len(a) != len(b) || len(a) < 2 {
+ return false
+ }
+ for i := range a {
+ if a[i].Offset != b[i].Offset || a[i].Size != b[i].Size || a[i].ETag != b[i].ETag {
+ return false
+ }
+ }
+ return true
+}
+
+// usesChunkETag reports whether filer.ETag falls back to ETagChunks for entry,
+// i.e. no stored attr.Md5 whole-content hash is available. Mirrors filer.ETag's
+// own condition so it agrees with how the compared ETag was computed.
+func usesChunkETag(entry *filer_pb.Entry) bool {
+ return entry.Attributes == nil || entry.Attributes.Md5 == nil
+}
+
+// sortedSimpleChunks returns chunks sorted by (offset, ETag) if they are a
+// manifest-free, non-overlapping set whose visible bytes are fully determined
+// by the list and every chunk carries a usable per-chunk ETag; otherwise nil.
+// Overlapping ranges would require resolving the visible interval by chunk
+// timestamp, which this fast path does not attempt.
+func sortedSimpleChunks(chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk {
+ sorted := make([]*filer_pb.FileChunk, len(chunks))
+ copy(sorted, chunks)
+ sort.Slice(sorted, func(i, j int) bool {
+ if sorted[i].Offset != sorted[j].Offset {
+ return sorted[i].Offset < sorted[j].Offset
+ }
+ return sorted[i].ETag < sorted[j].ETag
+ })
+ prevEnd := int64(0)
+ for i, c := range sorted {
+ if c.IsChunkManifest {
+ return nil
+ }
+ // An empty or undecodable ETag is not a content fingerprint, so
+ // element-wise (offset, size, ETag) equality cannot prove the bytes
+ // match — decline rather than assert a false content-equality.
+ if len(util.Base64Md5ToBytes(c.ETag)) == 0 {
+ return nil
+ }
+ if i > 0 && c.Offset < prevEnd {
+ return nil
+ }
+ prevEnd = c.Offset + int64(c.Size)
+ }
+ return sorted
+}
+
// mtimeRelation classifies B.mtime vs A.mtime. Both entries must be non-nil.
// Returns relation, absolute delta in seconds, and human-readable string.
func mtimeRelation(entryA, entryB *filer_pb.Entry) (relation, deltaStr string, deltaSec int64) {
@@ -557,11 +647,19 @@ func reportDiff(diffType verifyDiffType, dir string, entryA, entryB *filer_pb.En
result.sizeMismatch.Add(1)
case diffETagMismatch:
result.etagMismatch.Add(1)
+ case diffChunkReorder:
+ result.chunkReorder.Add(1)
}
if result.jsonOutput {
writeJSONDiff(result, diffType, dir, entryA, entryB)
} else {
+ // A chunk reorder is content-equal, not a real diff: keep it out of the
+ // default text report and list it only at -v=1. JSON always emits it so
+ // the per-record stream matches the summary count.
+ if diffType == diffChunkReorder && !glog.V(1) {
+ return
+ }
writeTextDiff(result, diffType, dir, entryA, entryB)
}
}
@@ -590,6 +688,9 @@ func writeTextDiff(result *VerifyResult, diffType verifyDiffType, dir string, en
ann := annotation(entryA, entryB)
fmt.Fprintf(os.Stdout, "[ETAG_MISMATCH] %s (a=%s, b=%s%s)\n",
entryPath, filer.ETag(entryA), filer.ETag(entryB), ann)
+ case diffChunkReorder:
+ fmt.Fprintf(os.Stdout, "[CHUNK_REORDER] %s (a=%s, b=%s, content-equal: chunks differ only in slice order)\n",
+ entryPath, filer.ETag(entryA), filer.ETag(entryB))
}
}
@@ -640,6 +741,14 @@ func writeJSONDiff(result *VerifyResult, diffType verifyDiffType, dir string, en
rec.MtimeRelation = relation
rec.MtimeDelta = delta
rec.Hint = hintFor(relation)
+ case diffChunkReorder:
+ rec.Type = "CHUNK_REORDER"
+ rec.A = toEntryRecord(entryA)
+ rec.B = toEntryRecord(entryB)
+ relation, delta, _ := mtimeRelation(entryA, entryB)
+ rec.MtimeRelation = relation
+ rec.MtimeDelta = delta
+ // no hint: content-equal, no action needed
}
writeJSONLine(result, rec)
diff --git a/weed/command/filer_sync_verify_test.go b/weed/command/filer_sync_verify_test.go
index 1fc6a6b241..c26014bccd 100644
--- a/weed/command/filer_sync_verify_test.go
+++ b/weed/command/filer_sync_verify_test.go
@@ -4,11 +4,14 @@ import (
"context"
"fmt"
"io"
+ "os"
+ "strings"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+ "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
@@ -101,6 +104,34 @@ func verifyDirEntry(name string) *filer_pb.Entry {
return &filer_pb.Entry{Name: name, IsDirectory: true}
}
+// verifyChunk builds a FileChunk whose ETag is the base64 MD5 of seed, so
+// filer.ETagChunks (which base64-decodes each chunk ETag) can hash it. Distinct
+// seeds yield distinct per-chunk MD5s.
+func verifyChunk(offset int64, size uint64, seed string) *filer_pb.FileChunk {
+ return &filer_pb.FileChunk{
+ Offset: offset,
+ Size: size,
+ ETag: util.Base64Encode(util.Md5([]byte(seed))),
+ }
+}
+
+// verifyChunkedEntry builds a chunk-backed entry with no stored attr.Md5, so
+// its file ETag is derived from ETagChunks (the order-sensitive path). FileSize
+// is the max chunk end, matching filer.FileSize, so size comparison passes.
+func verifyChunkedEntry(name string, chunks []*filer_pb.FileChunk) *filer_pb.Entry {
+ var total uint64
+ for _, c := range chunks {
+ if end := uint64(c.Offset) + c.Size; end > total {
+ total = end
+ }
+ }
+ return &filer_pb.Entry{
+ Name: name,
+ Attributes: &filer_pb.FuseAttributes{FileSize: total}, // Md5 nil → ETagChunks path
+ Chunks: chunks,
+ }
+}
+
// --- tests ---
// TestVerifySyncMissingFile confirms that a file present in A but absent in B
@@ -280,6 +311,271 @@ func TestVerifySyncETagMismatch(t *testing.T) {
}
}
+// TestVerifySyncChunkReorder confirms that two entries holding the SAME
+// chunks (same offsets, same per-chunk MD5s) in a DIFFERENT slice order are
+// classified as CHUNK_REORDER (content-equal), not ETAG_MISMATCH, and are
+// therefore not counted as errors. This is the S3-multipart vs filer.backup
+// reordering false positive.
+func TestVerifySyncChunkReorder(t *testing.T) {
+ c0 := verifyChunk(0, 100, "chunk-0")
+ c1 := verifyChunk(100, 100, "chunk-1")
+ c2 := verifyChunk(200, 100, "chunk-2")
+
+ // A stores [c0,c1,c2]; B stores a permutation [c2,c0,c1] of the same chunks.
+ clientA := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c0, c1, c2})},
+ },
+ }
+ clientB := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c2, c0, c1})},
+ },
+ }
+
+ result := &VerifyResult{}
+ sem := make(chan struct{}, verifySyncConcurrency)
+ if err := compareDirectory(context.Background(), clientA, clientB,
+ "/root", "/root", false, time.Time{}, sem, result); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if got := result.chunkReorder.Load(); got != 1 {
+ t.Errorf("chunkReorder = %d, want 1", got)
+ }
+ if got := result.etagMismatch.Load(); got != 0 {
+ t.Errorf("etagMismatch = %d, want 0 (reordered chunks are content-equal)", got)
+ }
+ if got := result.sizeMismatch.Load(); got != 0 {
+ t.Errorf("sizeMismatch = %d, want 0", got)
+ }
+}
+
+// TestVerifySyncGenuineChunkDivergence confirms that when one chunk's content
+// actually differs (same offsets and count, different per-chunk MD5), the file
+// stays classified as ETAG_MISMATCH and is NOT downgraded to CHUNK_REORDER.
+func TestVerifySyncGenuineChunkDivergence(t *testing.T) {
+ clientA := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{
+ verifyChunk(0, 100, "chunk-0"),
+ verifyChunk(100, 100, "chunk-1"),
+ verifyChunk(200, 100, "chunk-2"),
+ })},
+ },
+ }
+ clientB := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{
+ verifyChunk(0, 100, "chunk-0"),
+ verifyChunk(100, 100, "chunk-1"),
+ verifyChunk(200, 100, "chunk-2-DIFFERENT"), // real content divergence
+ })},
+ },
+ }
+
+ result := &VerifyResult{}
+ sem := make(chan struct{}, verifySyncConcurrency)
+ if err := compareDirectory(context.Background(), clientA, clientB,
+ "/root", "/root", false, time.Time{}, sem, result); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if got := result.etagMismatch.Load(); got != 1 {
+ t.Errorf("etagMismatch = %d, want 1 (genuine chunk content divergence)", got)
+ }
+ if got := result.chunkReorder.Load(); got != 0 {
+ t.Errorf("chunkReorder = %d, want 0 (content actually differs)", got)
+ }
+}
+
+// TestVerifySyncChunkCountDiffStaysEtagMismatch confirms that a differing chunk
+// count with an equal file size (e.g. differently split content) is NOT treated
+// as a reordering — it stays ETAG_MISMATCH.
+func TestVerifySyncChunkCountDiffStaysEtagMismatch(t *testing.T) {
+ clientA := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{
+ verifyChunk(0, 100, "a"),
+ verifyChunk(100, 100, "b"),
+ verifyChunk(200, 100, "c"),
+ })},
+ },
+ }
+ clientB := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{
+ verifyChunk(0, 150, "x"),
+ verifyChunk(150, 150, "y"),
+ })},
+ },
+ }
+
+ result := &VerifyResult{}
+ sem := make(chan struct{}, verifySyncConcurrency)
+ if err := compareDirectory(context.Background(), clientA, clientB,
+ "/root", "/root", false, time.Time{}, sem, result); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if got := result.sizeMismatch.Load(); got != 0 {
+ t.Errorf("sizeMismatch = %d, want 0 (both total 300 bytes)", got)
+ }
+ if got := result.etagMismatch.Load(); got != 1 {
+ t.Errorf("etagMismatch = %d, want 1", got)
+ }
+ if got := result.chunkReorder.Load(); got != 0 {
+ t.Errorf("chunkReorder = %d, want 0", got)
+ }
+}
+
+// TestVerifySyncDuplicateOffsetStaysEtagMismatch confirms that overlapping
+// chunks at the same offset — where the visible bytes are resolved by chunk
+// timestamp, not by the raw chunk list — are NOT fast-pathed to CHUNK_REORDER.
+// Both sides hold the same two ETags at offset 0 in a different order, so the
+// file ETags differ; the visible content is ambiguous from the list alone, so
+// this must stay ETAG_MISMATCH.
+func TestVerifySyncDuplicateOffsetStaysEtagMismatch(t *testing.T) {
+ c1 := verifyChunk(0, 100, "v1")
+ c2 := verifyChunk(0, 100, "v2") // same offset → overlap
+
+ clientA := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c1, c2})},
+ },
+ }
+ clientB := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c2, c1})},
+ },
+ }
+
+ result := &VerifyResult{}
+ sem := make(chan struct{}, verifySyncConcurrency)
+ if err := compareDirectory(context.Background(), clientA, clientB,
+ "/root", "/root", false, time.Time{}, sem, result); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if got := result.etagMismatch.Load(); got != 1 {
+ t.Errorf("etagMismatch = %d, want 1 (overlapping offsets are not a safe reorder)", got)
+ }
+ if got := result.chunkReorder.Load(); got != 0 {
+ t.Errorf("chunkReorder = %d, want 0", got)
+ }
+}
+
+// TestVerifySyncManifestChunkStaysEtagMismatch confirms that entries containing
+// a manifest chunk are not fast-pathed to CHUNK_REORDER: manifest chunks
+// represent compacted, possibly overlapping history that this check does not
+// resolve.
+func TestVerifySyncManifestChunkStaysEtagMismatch(t *testing.T) {
+ manifest := func(offset int64, size uint64, seed string) *filer_pb.FileChunk {
+ c := verifyChunk(offset, size, seed)
+ c.IsChunkManifest = true
+ return c
+ }
+ a0, a1 := manifest(0, 100, "m0"), verifyChunk(100, 100, "m1")
+ clientA := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{a0, a1})},
+ },
+ }
+ clientB := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{a1, a0})},
+ },
+ }
+
+ result := &VerifyResult{}
+ sem := make(chan struct{}, verifySyncConcurrency)
+ if err := compareDirectory(context.Background(), clientA, clientB,
+ "/root", "/root", false, time.Time{}, sem, result); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if got := result.etagMismatch.Load(); got != 1 {
+ t.Errorf("etagMismatch = %d, want 1 (manifest chunks are not fast-pathed)", got)
+ }
+ if got := result.chunkReorder.Load(); got != 0 {
+ t.Errorf("chunkReorder = %d, want 0", got)
+ }
+}
+
+// TestVerifySyncEmptyChunkETagStaysEtagMismatch confirms that a chunk with an
+// empty per-chunk ETag is never fast-pathed to CHUNK_REORDER: an empty ETag is
+// not a content fingerprint, so (offset, size, ETag) equality cannot prove the
+// bytes match. The two sides reorder their non-empty chunks so the file ETags
+// differ and the reorder check is reached, but the empty-ETag chunk in the
+// middle must keep it ETAG_MISMATCH rather than assert a false content-equality.
+func TestVerifySyncEmptyChunkETagStaysEtagMismatch(t *testing.T) {
+ c0 := verifyChunk(0, 100, "e0")
+ empty := &filer_pb.FileChunk{Offset: 100, Size: 100} // no ETag → not a fingerprint
+ c2 := verifyChunk(200, 100, "e2")
+
+ clientA := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c0, empty, c2})},
+ },
+ }
+ clientB := &verifyTestFilerClient{
+ entriesByDir: map[string][]*filer_pb.Entry{
+ "/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c2, c0, empty})},
+ },
+ }
+
+ result := &VerifyResult{}
+ sem := make(chan struct{}, verifySyncConcurrency)
+ if err := compareDirectory(context.Background(), clientA, clientB,
+ "/root", "/root", false, time.Time{}, sem, result); err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if got := result.etagMismatch.Load(); got != 1 {
+ t.Errorf("etagMismatch = %d, want 1 (empty per-chunk ETag is not a content fingerprint)", got)
+ }
+ if got := result.chunkReorder.Load(); got != 0 {
+ t.Errorf("chunkReorder = %d, want 0", got)
+ }
+}
+
+// captureStdout runs fn with os.Stdout redirected to a pipe and returns what it
+// wrote. Tests that assert on emitted diff records use it.
+func captureStdout(t *testing.T, fn func()) string {
+ t.Helper()
+ orig := os.Stdout
+ r, w, err := os.Pipe()
+ if err != nil {
+ t.Fatalf("os.Pipe: %v", err)
+ }
+ os.Stdout = w
+ fn()
+ w.Close()
+ os.Stdout = orig
+ data, err := io.ReadAll(r)
+ if err != nil {
+ t.Fatalf("read captured stdout: %v", err)
+ }
+ return string(data)
+}
+
+// TestVerifySyncChunkReorderJSONEmittedByDefault confirms that in JSON output
+// mode a CHUNK_REORDER record is emitted at default verbosity: the -v=1 gate
+// suppresses only the human text report, so the NDJSON per-record stream stays
+// consistent with the summary count a machine consumer reads.
+func TestVerifySyncChunkReorderJSONEmittedByDefault(t *testing.T) {
+ c0 := verifyChunk(0, 100, "chunk-0")
+ c1 := verifyChunk(100, 100, "chunk-1")
+ entryA := verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c0, c1})
+ entryB := verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c1, c0})
+
+ result := &VerifyResult{jsonOutput: true}
+ out := captureStdout(t, func() {
+ reportDiff(diffChunkReorder, "/root", entryA, entryB, result)
+ })
+
+ if got := result.chunkReorder.Load(); got != 1 {
+ t.Fatalf("chunkReorder = %d, want 1", got)
+ }
+ if !strings.Contains(out, `"type":"CHUNK_REORDER"`) {
+ t.Errorf("JSON output missing CHUNK_REORDER record at default verbosity; got %q", out)
+ }
+}
+
// TestVerifySyncCutoffTime verifies that entries newer than cutoffTime are
// skipped in both the A-only (MISSING) and B-only (ONLY_IN_B) branches.
func TestVerifySyncCutoffTime(t *testing.T) {