SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package command
import (
"context"
"fmt"
"io"
"os"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
)
// --- stream / inner-client / outer-client mocks ---
type verifyTestStream struct {
entries []*filer_pb.Entry
idx int
recvErr error // returned after all entries instead of io.EOF, if set
}
func (s *verifyTestStream) Recv() (*filer_pb.ListEntriesResponse, error) {
if s.idx >= len(s.entries) {
if s.recvErr != nil {
return nil, s.recvErr
}
return nil, io.EOF
}
resp := &filer_pb.ListEntriesResponse{Entry: s.entries[s.idx]}
s.idx++
return resp, nil
}
func (s *verifyTestStream) Header() (metadata.MD, error) { return metadata.MD{}, nil }
func (s *verifyTestStream) Trailer() metadata.MD { return metadata.MD{} }
func (s *verifyTestStream) CloseSend() error { return nil }
func (s *verifyTestStream) Context() context.Context { return context.Background() }
func (s *verifyTestStream) SendMsg(_ any) error { return nil }
func (s *verifyTestStream) RecvMsg(_ any) error { return nil }
// verifyTestInnerClient is the SeaweedFilerClient passed to fn inside WithFilerClient.
type verifyTestInnerClient struct {
filer_pb.SeaweedFilerClient // embed for unimplemented RPCs
entriesByDir map[string][]*filer_pb.Entry
recvErr error // injected listing error, if set
}
func (c *verifyTestInnerClient) ListEntries(_ context.Context, in *filer_pb.ListEntriesRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
return &verifyTestStream{entries: c.entriesByDir[in.Directory], recvErr: c.recvErr}, nil
}
// verifyTestFilerClient implements filer_pb.FilerClient and tracks concurrent
// WithFilerClient invocations to let tests verify the global concurrency bound.
// inFlight/peakFlight use atomic.Int64 (not bare int64 + sync/atomic funcs) so
// the fields are 8-byte aligned even on 32-bit platforms — bare int64 fields
// after a pointer-sized field panic with "unaligned 64-bit atomic operation"
// on 386/arm/mips.
type verifyTestFilerClient struct {
entriesByDir map[string][]*filer_pb.Entry
inFlight atomic.Int64
peakFlight atomic.Int64
delay time.Duration
onList func() // called at the start of each listing, if set
recvErr error // injected listing error, surfaced after entries
}
func (c *verifyTestFilerClient) WithFilerClient(_ bool, fn func(filer_pb.SeaweedFilerClient) error) error {
// track peak concurrent in-flight listings
n := c.inFlight.Add(1)
defer c.inFlight.Add(-1)
for {
peak := c.peakFlight.Load()
if n <= peak || c.peakFlight.CompareAndSwap(peak, n) {
break
}
}
if c.onList != nil {
c.onList()
}
if c.delay > 0 {
time.Sleep(c.delay)
}
return fn(&verifyTestInnerClient{entriesByDir: c.entriesByDir, recvErr: c.recvErr})
}
func (c *verifyTestFilerClient) AdjustedUrl(_ *filer_pb.Location) string { return "" }
func (c *verifyTestFilerClient) GetDataCenter() string { return "" }
// --- entry helpers ---
func verifyFileEntry(name string, size uint64) *filer_pb.Entry {
return &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{FileSize: size},
}
}
func verifyDirEntry(name string) *filer_pb.Entry {
return &filer_pb.Entry{Name: name, IsDirectory: true}
}
// verifyChunk builds a FileChunk whose ETag is the base64 MD5 of seed, so
// filer.ETagChunks (which base64-decodes each chunk ETag) can hash it. Distinct
// seeds yield distinct per-chunk MD5s.
func verifyChunk(offset int64, size uint64, seed string) *filer_pb.FileChunk {
return &filer_pb.FileChunk{
Offset: offset,
Size: size,
ETag: util.Base64Encode(util.Md5([]byte(seed))),
}
}
// verifyChunkedEntry builds a chunk-backed entry with no stored attr.Md5, so
// its file ETag is derived from ETagChunks (the order-sensitive path). FileSize
// is the max chunk end, matching filer.FileSize, so size comparison passes.
func verifyChunkedEntry(name string, chunks []*filer_pb.FileChunk) *filer_pb.Entry {
var total uint64
for _, c := range chunks {
if end := uint64(c.Offset) + c.Size; end > total {
total = end
}
}
return &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{FileSize: total}, // Md5 nil → ETagChunks path
Chunks: chunks,
}
}
// --- tests ---
// TestVerifySyncMissingFile confirms that a file present in A but absent in B
// is counted as missing.
func TestVerifySyncMissingFile(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyFileEntry("file.txt", 100)},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.missingCount.Load(); got != 1 {
t.Errorf("missingCount = %d, want 1", got)
}
if got := result.sizeMismatch.Load(); got != 0 {
t.Errorf("sizeMismatch = %d, want 0", got)
}
}
// TestVerifySyncOnlyInB confirms that a file present only in B is counted
// (non-active-passive mode) or ignored (active-passive mode).
func TestVerifySyncOnlyInB(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/root": {}},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyFileEntry("extra.txt", 50)},
},
}
t.Run("bidirectional", func(t *testing.T) {
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.onlyInB.Load(); got != 1 {
t.Errorf("onlyInB = %d, want 1", got)
}
})
t.Run("active-passive ignores onlyInB", func(t *testing.T) {
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", true, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.onlyInB.Load(); got != 0 {
t.Errorf("onlyInB = %d, want 0 in active-passive mode", got)
}
})
}
// TestVerifySyncSizeMismatch confirms that a file with differing sizes is
// counted as a size mismatch and not as missing.
func TestVerifySyncSizeMismatch(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyFileEntry("data.bin", 1024)},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyFileEntry("data.bin", 512)},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.sizeMismatch.Load(); got != 1 {
t.Errorf("sizeMismatch = %d, want 1", got)
}
if got := result.missingCount.Load(); got != 0 {
t.Errorf("missingCount = %d, want 0", got)
}
}
// TestVerifySyncConcurrencyBound verifies that the shared semaphore keeps peak
// concurrent filer listings at or below verifySyncConcurrency at all times.
// A 5ms delay per WithFilerClient call makes the concurrency overlap observable.
func TestVerifySyncConcurrencyBound(t *testing.T) {
// Wide, shallow tree: root with 20 identical subdirectories.
const fanout = 20
entriesA := make(map[string][]*filer_pb.Entry)
entriesB := make(map[string][]*filer_pb.Entry)
rootDirs := make([]*filer_pb.Entry, fanout)
for i := range fanout {
name := fmt.Sprintf("sub%02d", i)
rootDirs[i] = verifyDirEntry(name)
entriesA["/root/"+name] = []*filer_pb.Entry{verifyFileEntry("f.txt", 10)}
entriesB["/root/"+name] = []*filer_pb.Entry{verifyFileEntry("f.txt", 10)}
}
entriesA["/root"] = rootDirs
entriesB["/root"] = rootDirs
clientA := &verifyTestFilerClient{entriesByDir: entriesA, delay: 5 * time.Millisecond}
clientB := &verifyTestFilerClient{entriesByDir: entriesB, delay: 5 * time.Millisecond}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.missingCount.Load() != 0 || result.sizeMismatch.Load() != 0 {
t.Errorf("unexpected diffs in identical tree")
}
if peak := clientA.peakFlight.Load(); peak > verifySyncConcurrency {
t.Errorf("clientA peak concurrent listings = %d, want ≤ %d (verifySyncConcurrency)",
peak, verifySyncConcurrency)
}
if peak := clientB.peakFlight.Load(); peak > verifySyncConcurrency {
t.Errorf("clientB peak concurrent listings = %d, want ≤ %d (verifySyncConcurrency)",
peak, verifySyncConcurrency)
}
}
// TestVerifySyncETagMismatch confirms that two files with the same size but
// different Md5 checksums are counted as an ETag mismatch, not a size mismatch.
func TestVerifySyncETagMismatch(t *testing.T) {
newEntry := func(name string, md5 []byte) *filer_pb.Entry {
return &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{
FileSize: 100,
Md5: md5,
},
}
}
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {newEntry("data.bin", []byte{0x11, 0x22, 0x33})},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {newEntry("data.bin", []byte{0x44, 0x55, 0x66})},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.etagMismatch.Load(); got != 1 {
t.Errorf("etagMismatch = %d, want 1", got)
}
if got := result.sizeMismatch.Load(); got != 0 {
t.Errorf("sizeMismatch = %d, want 0 (same size should not trigger size mismatch)", got)
}
}
// TestVerifySyncChunkReorder confirms that two entries holding the SAME
// chunks (same offsets, same per-chunk MD5s) in a DIFFERENT slice order are
// classified as CHUNK_REORDER (content-equal), not ETAG_MISMATCH, and are
// therefore not counted as errors. This is the S3-multipart vs filer.backup
// reordering false positive.
func TestVerifySyncChunkReorder(t *testing.T) {
c0 := verifyChunk(0, 100, "chunk-0")
c1 := verifyChunk(100, 100, "chunk-1")
c2 := verifyChunk(200, 100, "chunk-2")
// A stores [c0,c1,c2]; B stores a permutation [c2,c0,c1] of the same chunks.
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c0, c1, c2})},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c2, c0, c1})},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.chunkReorder.Load(); got != 1 {
t.Errorf("chunkReorder = %d, want 1", got)
}
if got := result.etagMismatch.Load(); got != 0 {
t.Errorf("etagMismatch = %d, want 0 (reordered chunks are content-equal)", got)
}
if got := result.sizeMismatch.Load(); got != 0 {
t.Errorf("sizeMismatch = %d, want 0", got)
}
}
// TestVerifySyncGenuineChunkDivergence confirms that when one chunk's content
// actually differs (same offsets and count, different per-chunk MD5), the file
// stays classified as ETAG_MISMATCH and is NOT downgraded to CHUNK_REORDER.
func TestVerifySyncGenuineChunkDivergence(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{
verifyChunk(0, 100, "chunk-0"),
verifyChunk(100, 100, "chunk-1"),
verifyChunk(200, 100, "chunk-2"),
})},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{
verifyChunk(0, 100, "chunk-0"),
verifyChunk(100, 100, "chunk-1"),
verifyChunk(200, 100, "chunk-2-DIFFERENT"), // real content divergence
})},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.etagMismatch.Load(); got != 1 {
t.Errorf("etagMismatch = %d, want 1 (genuine chunk content divergence)", got)
}
if got := result.chunkReorder.Load(); got != 0 {
t.Errorf("chunkReorder = %d, want 0 (content actually differs)", got)
}
}
// TestVerifySyncChunkCountDiffStaysEtagMismatch confirms that a differing chunk
// count with an equal file size (e.g. differently split content) is NOT treated
// as a reordering — it stays ETAG_MISMATCH.
func TestVerifySyncChunkCountDiffStaysEtagMismatch(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{
verifyChunk(0, 100, "a"),
verifyChunk(100, 100, "b"),
verifyChunk(200, 100, "c"),
})},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{
verifyChunk(0, 150, "x"),
verifyChunk(150, 150, "y"),
})},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.sizeMismatch.Load(); got != 0 {
t.Errorf("sizeMismatch = %d, want 0 (both total 300 bytes)", got)
}
if got := result.etagMismatch.Load(); got != 1 {
t.Errorf("etagMismatch = %d, want 1", got)
}
if got := result.chunkReorder.Load(); got != 0 {
t.Errorf("chunkReorder = %d, want 0", got)
}
}
// TestVerifySyncDuplicateOffsetStaysEtagMismatch confirms that overlapping
// chunks at the same offset — where the visible bytes are resolved by chunk
// timestamp, not by the raw chunk list — are NOT fast-pathed to CHUNK_REORDER.
// Both sides hold the same two ETags at offset 0 in a different order, so the
// file ETags differ; the visible content is ambiguous from the list alone, so
// this must stay ETAG_MISMATCH.
func TestVerifySyncDuplicateOffsetStaysEtagMismatch(t *testing.T) {
c1 := verifyChunk(0, 100, "v1")
c2 := verifyChunk(0, 100, "v2") // same offset → overlap
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c1, c2})},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c2, c1})},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.etagMismatch.Load(); got != 1 {
t.Errorf("etagMismatch = %d, want 1 (overlapping offsets are not a safe reorder)", got)
}
if got := result.chunkReorder.Load(); got != 0 {
t.Errorf("chunkReorder = %d, want 0", got)
}
}
// TestVerifySyncManifestChunkStaysEtagMismatch confirms that entries containing
// a manifest chunk are not fast-pathed to CHUNK_REORDER: manifest chunks
// represent compacted, possibly overlapping history that this check does not
// resolve.
func TestVerifySyncManifestChunkStaysEtagMismatch(t *testing.T) {
manifest := func(offset int64, size uint64, seed string) *filer_pb.FileChunk {
c := verifyChunk(offset, size, seed)
c.IsChunkManifest = true
return c
}
a0, a1 := manifest(0, 100, "m0"), verifyChunk(100, 100, "m1")
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{a0, a1})},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{a1, a0})},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.etagMismatch.Load(); got != 1 {
t.Errorf("etagMismatch = %d, want 1 (manifest chunks are not fast-pathed)", got)
}
if got := result.chunkReorder.Load(); got != 0 {
t.Errorf("chunkReorder = %d, want 0", got)
}
}
// TestVerifySyncEmptyChunkETagStaysEtagMismatch confirms that a chunk with an
// empty per-chunk ETag is never fast-pathed to CHUNK_REORDER: an empty ETag is
// not a content fingerprint, so (offset, size, ETag) equality cannot prove the
// bytes match. The two sides reorder their non-empty chunks so the file ETags
// differ and the reorder check is reached, but the empty-ETag chunk in the
// middle must keep it ETAG_MISMATCH rather than assert a false content-equality.
func TestVerifySyncEmptyChunkETagStaysEtagMismatch(t *testing.T) {
c0 := verifyChunk(0, 100, "e0")
empty := &filer_pb.FileChunk{Offset: 100, Size: 100} // no ETag → not a fingerprint
c2 := verifyChunk(200, 100, "e2")
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c0, empty, c2})},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/root": {verifyChunkedEntry("data.bin", []*filer_pb.FileChunk{c2, c0, empty})},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.etagMismatch.Load(); got != 1 {
t.Errorf("etagMismatch = %d, want 1 (empty per-chunk ETag is not a content fingerprint)", got)
}
if got := result.chunkReorder.Load(); got != 0 {
t.Errorf("chunkReorder = %d, want 0", got)
}
}
// captureStdout runs fn with os.Stdout redirected to a pipe and returns what it
// wrote. Tests that assert on emitted diff records use it.
func captureStdout(t *testing.T, fn func()) string {
t.Helper()
orig := os.Stdout
r, w, err := os.Pipe()
if err != nil {
t.Fatalf("os.Pipe: %v", err)
}
os.Stdout = w
fn()
w.Close()
os.Stdout = orig
data, err := io.ReadAll(r)
if err != nil {
t.Fatalf("read captured stdout: %v", err)
}
return string(data)
}
// TestVerifySyncChunkReorderJSONEmittedByDefault confirms that in JSON output
// mode a CHUNK_REORDER record is emitted at default verbosity: the -v=1 gate
// suppresses only the human text report, so the NDJSON per-record stream stays
// consistent with the summary count a machine consumer reads.
func TestVerifySyncChunkReorderJSONEmittedByDefault(t *testing.T) {
c0 := verifyChunk(0, 100, "chunk-0")
c1 := verifyChunk(100, 100, "chunk-1")
entryA := verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c0, c1})
entryB := verifyChunkedEntry("model.bin", []*filer_pb.FileChunk{c1, c0})
result := &VerifyResult{jsonOutput: true}
out := captureStdout(t, func() {
reportDiff(diffChunkReorder, "/root", entryA, entryB, result)
})
if got := result.chunkReorder.Load(); got != 1 {
t.Fatalf("chunkReorder = %d, want 1", got)
}
if !strings.Contains(out, `"type":"CHUNK_REORDER"`) {
t.Errorf("JSON output missing CHUNK_REORDER record at default verbosity; got %q", out)
}
}
// TestVerifySyncCutoffTime verifies that entries newer than cutoffTime are
// skipped in both the A-only (MISSING) and B-only (ONLY_IN_B) branches.
func TestVerifySyncCutoffTime(t *testing.T) {
cutoff := time.Unix(1000, 0)
recentEntry := func(name string) *filer_pb.Entry {
return &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{FileSize: 10, Mtime: 2000}, // > cutoff
}
}
oldEntry := func(name string) *filer_pb.Entry {
return &filer_pb.Entry{
Name: name,
Attributes: &filer_pb.FuseAttributes{FileSize: 10, Mtime: 500}, // < cutoff
}
}
t.Run("A-only recent file is skipped, not reported missing", func(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {recentEntry("new.txt")}},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {}},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/", "/", false, cutoff, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.skippedRecent.Load(); got != 1 {
t.Errorf("skippedRecent = %d, want 1", got)
}
if got := result.missingCount.Load(); got != 0 {
t.Errorf("missingCount = %d, want 0 (recent file should be skipped)", got)
}
})
t.Run("A-only old file is reported missing", func(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {oldEntry("old.txt")}},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {}},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/", "/", false, cutoff, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.missingCount.Load(); got != 1 {
t.Errorf("missingCount = %d, want 1", got)
}
})
t.Run("B-only recent file is skipped, not reported as ONLY_IN_B", func(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {}},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {recentEntry("new.txt")}},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/", "/", false, cutoff, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.skippedRecent.Load(); got != 1 {
t.Errorf("skippedRecent = %d, want 1", got)
}
if got := result.onlyInB.Load(); got != 0 {
t.Errorf("onlyInB = %d, want 0 (recent B-only file should be skipped)", got)
}
})
t.Run("B-only old file is reported as ONLY_IN_B", func(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {}},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {oldEntry("old.txt")}},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/", "/", false, cutoff, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.onlyInB.Load(); got != 1 {
t.Errorf("onlyInB = %d, want 1", got)
}
})
}
// TestVerifySyncCutoffMatchedFileBSideRecent verifies that when matched-name
// files differ but only the B side is recently modified, the comparison is
// skipped (sync-lag tolerance) rather than reporting a spurious mismatch.
func TestVerifySyncCutoffMatchedFileBSideRecent(t *testing.T) {
cutoff := time.Unix(1000, 0)
entry := func(size uint64, mtime int64) *filer_pb.Entry {
return &filer_pb.Entry{
Name: "data.bin",
Attributes: &filer_pb.FuseAttributes{FileSize: size, Mtime: mtime},
}
}
// A is old (size 100), B is recently rewritten with a different size.
// Without the B-side cutoff check this would surface as SIZE_MISMATCH.
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {entry(100, 500)}},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/": {entry(200, 2000)}},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/", "/", false, cutoff, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.skippedRecent.Load(); got != 1 {
t.Errorf("skippedRecent = %d, want 1 (B-side recent should skip)", got)
}
if got := result.sizeMismatch.Load(); got != 0 {
t.Errorf("sizeMismatch = %d, want 0 (recent B should not surface as mismatch)", got)
}
}
// TestVerifySyncMissingDirRecursesEvenWithRecentMtime verifies that a
// directory missing in B with a recent mtime still has its subtree walked,
// so older missing files inside are reported. A recent child write can bump
// the parent mtime even though older missing files exist underneath.
func TestVerifySyncMissingDirRecursesEvenWithRecentMtime(t *testing.T) {
cutoff := time.Unix(1000, 0)
recentDir := &filer_pb.Entry{
Name: "subdir",
IsDirectory: true,
Attributes: &filer_pb.FuseAttributes{Mtime: 2000}, // > cutoff
}
oldChild := &filer_pb.Entry{
Name: "old.txt",
Attributes: &filer_pb.FuseAttributes{FileSize: 10, Mtime: 500}, // < cutoff
}
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/": {recentDir},
"/subdir": {oldChild},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/": {},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/", "/", false, cutoff, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Expect: directory MISSING + recursed-old-file MISSING = 2 missing.
if got := result.missingCount.Load(); got != 2 {
t.Errorf("missingCount = %d, want 2 (recent dir + old child inside)", got)
}
if got := result.skippedRecent.Load(); got != 0 {
t.Errorf("skippedRecent = %d, want 0 (dir mtime should not gate recursion)", got)
}
}
// TestVerifySyncRootPath is a regression test for the path.Join fix.
// fmt.Sprintf("%s/%s", "/", name) produced "//name"; path.Join produces "/name".
// This test walks from "/" and verifies the child directory is found and
// compared correctly (not silently skipped due to a malformed path).
func TestVerifySyncRootPath(t *testing.T) {
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/": {verifyDirEntry("data")},
"/data": {verifyFileEntry("file.txt", 42)},
},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{
"/": {verifyDirEntry("data")},
"/data": {verifyFileEntry("file.txt", 42)},
},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/", "/", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if result.missingCount.Load() != 0 || result.sizeMismatch.Load() != 0 {
t.Errorf("identical trees from root should have no diffs: missing=%d size=%d",
result.missingCount.Load(), result.sizeMismatch.Load())
}
// 2 directories traversed: "/" and "/data"
if got := result.dirCount.Load(); got != 2 {
t.Errorf("dirCount = %d, want 2 (root + /data)", got)
}
// 1 file compared: /data/file.txt
if got := result.fileCount.Load(); got != 1 {
t.Errorf("fileCount = %d, want 1", got)
}
}
// TestVerifySyncNoDeadlockDeepTree ensures that a tree deeper than
// verifySyncConcurrency completes without deadlocking. With a per-call
// semaphore the walk would still complete (just with unbounded goroutines);
// this test mainly guards that the shared-semaphore release-before-recurse
// invariant holds — i.e. the walk finishes within the timeout.
func TestVerifySyncNoDeadlockDeepTree(t *testing.T) {
// Build a binary tree of depth 10 (well past verifySyncConcurrency=5).
const depth = 10
entriesA := make(map[string][]*filer_pb.Entry)
entriesB := make(map[string][]*filer_pb.Entry)
var buildTree func(path string, d int)
buildTree = func(path string, d int) {
if d == 0 {
entriesA[path] = []*filer_pb.Entry{verifyFileEntry("leaf.txt", 1)}
entriesB[path] = []*filer_pb.Entry{verifyFileEntry("leaf.txt", 1)}
return
}
children := []*filer_pb.Entry{verifyDirEntry("left"), verifyDirEntry("right")}
entriesA[path] = children
entriesB[path] = children
buildTree(path+"/left", d-1)
buildTree(path+"/right", d-1)
}
buildTree("/root", depth)
clientA := &verifyTestFilerClient{entriesByDir: entriesA}
clientB := &verifyTestFilerClient{entriesByDir: entriesB}
done := make(chan error, 1)
go func() {
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
done <- compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result)
}()
select {
case err := <-done:
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
case <-time.After(10 * time.Second):
t.Fatal("compareDirectory did not complete within 10s — possible deadlock")
}
}
// TestVerifySyncByteOrderSkew covers a collation skew between two filer
// versions: both return the SAME name set in DIFFERENT order. Filer A (older,
// pre-4.32) lists in locale collation (case-insensitive: lowercase mixes in
// among uppercase); filer B (4.32+, with PR #9824) lists in byte order
// (uppercase before lowercase). The mock returns each side's slice verbatim.
//
// Because compareDirectory sorts both sides client-side before merging, the two
// orders converge and no spurious diffs are reported. Were the sort removed, the
// streaming merge would desync and count 3 false MISSING + 3 false ONLY_IN_B.
func TestVerifySyncByteOrderSkew(t *testing.T) {
// locale order (case-insensitive): lowercase mixes in among uppercase.
localeOrder := []*filer_pb.Entry{
verifyFileEntry("sk-4", 10),
verifyFileEntry("sk-8", 10),
verifyFileEntry("sk-mmsJ", 10),
verifyFileEntry("sk-nFGE", 10),
verifyFileEntry("sk-RH0Z", 10),
verifyFileEntry("sk-Xp", 10),
verifyFileEntry("sk-Z06", 10),
}
// byte order: uppercase (R,X,Z) sort before lowercase (m,n).
byteOrder := []*filer_pb.Entry{
verifyFileEntry("sk-4", 10),
verifyFileEntry("sk-8", 10),
verifyFileEntry("sk-RH0Z", 10),
verifyFileEntry("sk-Xp", 10),
verifyFileEntry("sk-Z06", 10),
verifyFileEntry("sk-mmsJ", 10),
verifyFileEntry("sk-nFGE", 10),
}
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/root": localeOrder},
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/root": byteOrder},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got := result.missingCount.Load(); got != 0 {
t.Errorf("missingCount = %d, want 0", got)
}
if got := result.onlyInB.Load(); got != 0 {
t.Errorf("onlyInB = %d, want 0", got)
}
}
// TestVerifySyncSourcesListConcurrently guards against re-serializing the two
// directory listings: both sides buffer + sort, and their loads must run
// concurrently — not A fully then B. A shared gate releases only once both
// listings are in-flight at the same time; if they were sequential the first
// would block on the gate and time out.
func TestVerifySyncSourcesListConcurrently(t *testing.T) {
var started atomic.Int32
var timedOut atomic.Bool
release := make(chan struct{})
gate := func() {
if started.Add(1) == 2 {
close(release) // both listings reached the gate → proceed
}
select {
case <-release:
case <-time.After(3 * time.Second):
timedOut.Store(true) // only one listing ever in-flight → serialized
}
}
entries := []*filer_pb.Entry{verifyFileEntry("a", 1), verifyFileEntry("b", 1)}
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/root": entries},
onList: gate,
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/root": entries},
onList: gate,
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if timedOut.Load() {
t.Fatal("sources listed sequentially: both listings were not in-flight at once")
}
}
// TestVerifySyncListErrorNoBogusDiffs verifies that a listing failure aborts
// with the error and reports no differences. A side whose listing errors keeps
// only a partial, unsorted buffer; the error must be surfaced before the merge
// so those partial entries never produce spurious MISSING / ONLY_IN_B.
func TestVerifySyncListErrorNoBogusDiffs(t *testing.T) {
entries := []*filer_pb.Entry{
verifyFileEntry("a", 1),
verifyFileEntry("b", 1),
verifyFileEntry("c", 1),
}
// A errors mid-listing (partial buffer); B lists cleanly.
clientA := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/root": entries},
recvErr: fmt.Errorf("injected list failure"),
}
clientB := &verifyTestFilerClient{
entriesByDir: map[string][]*filer_pb.Entry{"/root": entries},
}
result := &VerifyResult{}
sem := make(chan struct{}, verifySyncConcurrency)
err := compareDirectory(context.Background(), clientA, clientB,
"/root", "/root", false, time.Time{}, sem, result)
if err == nil {
t.Fatal("expected a listing error, got nil")
}
if got := result.missingCount.Load(); got != 0 {
t.Errorf("missingCount = %d, want 0 (no bogus diffs on list error)", got)
}
if got := result.onlyInB.Load(); got != 0 {
t.Errorf("onlyInB = %d, want 0 (no bogus diffs on list error)", got)
}
if got := result.fileCount.Load(); got != 0 {
t.Errorf("fileCount = %d, want 0 (merge must not run on failed listing)", got)
}
}