SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package command
import (
"context"
"encoding/json"
"fmt"
"os"
"path"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
)
type SyncVerifyOptions struct {
filerA *string
filerB *string
aPath *string
bPath *string
aSecurity *string
bSecurity *string
isActivePassive *bool
modifiedTimeAgo *time.Duration
jsonOutput *bool
}
var syncVerifyOptions SyncVerifyOptions
func init() {
cmdFilerSyncVerify.Run = runFilerSyncVerify // break init cycle
syncVerifyOptions.filerA = cmdFilerSyncVerify.Flag.String("a", "", "filer A in one SeaweedFS cluster")
syncVerifyOptions.filerB = cmdFilerSyncVerify.Flag.String("b", "", "filer B in the other SeaweedFS cluster")
syncVerifyOptions.aPath = cmdFilerSyncVerify.Flag.String("a.path", "/", "directory to verify on filer A")
syncVerifyOptions.bPath = cmdFilerSyncVerify.Flag.String("b.path", "/", "directory to verify on filer B")
syncVerifyOptions.aSecurity = cmdFilerSyncVerify.Flag.String("a.security", "", "security.toml file for filer A when clusters use different certificates")
syncVerifyOptions.bSecurity = cmdFilerSyncVerify.Flag.String("b.security", "", "security.toml file for filer B when clusters use different certificates")
syncVerifyOptions.isActivePassive = cmdFilerSyncVerify.Flag.Bool("isActivePassive", false, "one directional comparison from A to B; entries only in B are not reported")
syncVerifyOptions.modifiedTimeAgo = cmdFilerSyncVerify.Flag.Duration("modifiedTimeAgo", 0, "only verify files modified before this duration ago (e.g. 1h) for sync-lag tolerance")
syncVerifyOptions.jsonOutput = cmdFilerSyncVerify.Flag.Bool("jsonOutput", false, "emit NDJSON output (one JSON object per line) for external tooling")
}
var cmdFilerSyncVerify = &Command{
UsageLine: "filer.sync.verify -a=<oneFilerHost>:<oneFilerPort> -b=<otherFilerHost>:<otherFilerPort>",
Short: "compare entries between two filers and report differences",
Long: `compare entries between two filers and report differences, then exit.
Useful for validating active/passive sync targets agree with the source.
Reports MISSING (in A but not in B), ONLY_IN_B (in B but not in A; suppressed
in active-passive mode), SIZE_MISMATCH, and ETAG_MISMATCH. Honors
-modifiedTimeAgo to skip recently-modified files (sync-lag tolerance) and
-isActivePassive for unidirectional comparison.
A chunk-derived ETag (no stored attr.Md5) can differ between clusters only
because the chunk slice was assembled in a different order, since ETagChunks
does not sort by offset. Such files are byte-identical and reported as
CHUNK_REORDER: not counted as errors, always counted in the summary, listed
only at -v=1.
Exits with code 0 on agreement, 2 on differences or operational errors.
`,
}
func runFilerSyncVerify(cmd *Command, args []string) bool {
*syncVerifyOptions.aSecurity = util.ResolvePath(*syncVerifyOptions.aSecurity)
*syncVerifyOptions.bSecurity = util.ResolvePath(*syncVerifyOptions.bSecurity)
util.LoadSecurityConfiguration()
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
grpcDialOptionA := grpcDialOption
grpcDialOptionB := grpcDialOption
if *syncVerifyOptions.aSecurity != "" {
var err error
if grpcDialOptionA, err = security.LoadClientTLSFromFile(*syncVerifyOptions.aSecurity, "grpc.client"); err != nil {
glog.Fatalf("load security config for filer A: %v", err)
}
}
if *syncVerifyOptions.bSecurity != "" {
var err error
if grpcDialOptionB, err = security.LoadClientTLSFromFile(*syncVerifyOptions.bSecurity, "grpc.client"); err != nil {
glog.Fatalf("load security config for filer B: %v", err)
}
}
filerA := pb.ServerAddress(*syncVerifyOptions.filerA)
filerB := pb.ServerAddress(*syncVerifyOptions.filerB)
if err := runVerifySync(filerA, filerB, *syncVerifyOptions.aPath, *syncVerifyOptions.bPath,
*syncVerifyOptions.isActivePassive, *syncVerifyOptions.modifiedTimeAgo,
*syncVerifyOptions.jsonOutput,
grpcDialOptionA, grpcDialOptionB); err != nil {
glog.Errorf("verify sync: %v", err)
os.Exit(2)
}
return true
}
// verifySyncConcurrency caps concurrent directory I/O across the entire
// recursive walk. A single shared semaphore is created in runVerifySync and
// passed down so the limit applies globally — a per-call semaphore would only
// cap concurrency per directory level, allowing fanout to grow as
// verifySyncConcurrency^depth on deep trees.
//
// Trade-off: higher values reduce wall time on wide trees by parallelizing
// listEntries RPCs, at the cost of more concurrent load on both filers and
// more memory from queued goroutines (each waiting goroutine ~2KB stack).
// Each compareDirectory holds a slot only for its own listing+compare phase
// and releases before recursing, so a parent never blocks waiting for
// children to acquire slots.
const verifySyncConcurrency = 5
// isTooRecent reports whether entry's mtime is past cutoff (sync-lag tolerance).
// Returns false when cutoff is zero or attributes are missing.
func isTooRecent(entry *filer_pb.Entry, cutoff time.Time) bool {
return !cutoff.IsZero() && entry != nil && entry.Attributes != nil && entry.Attributes.Mtime > cutoff.Unix()
}
type VerifyResult struct {
dirCount atomic.Int64
fileCount atomic.Int64
missingCount atomic.Int64
sizeMismatch atomic.Int64
etagMismatch atomic.Int64
chunkReorder atomic.Int64 // content-equal; chunks differ only in slice order
onlyInB atomic.Int64
skippedRecent atomic.Int64
// outputMu serializes writes to stdout. Multiple goroutines call
// reportDiff concurrently from compareDirectory worker pool.
outputMu sync.Mutex
jsonOutput bool
}
type verifyDiffType int
const (
diffMissing verifyDiffType = iota // in A but not in B
diffOnlyInB // in B but not in A
diffSizeMismatch // size differs
diffETagMismatch // etag differs
diffChunkReorder // etag differs but content is equal (chunk slice order only)
)
// diffRecord is the JSON Lines schema for a single diff entry.
type diffRecord struct {
Type string `json:"type"`
Path string `json:"path"`
IsDirectory bool `json:"isDirectory,omitempty"`
A *entryRecord `json:"a,omitempty"`
B *entryRecord `json:"b,omitempty"`
MtimeRelation string `json:"mtimeRelation,omitempty"` // EQUAL | A_NEWER | B_NEWER
MtimeDelta string `json:"mtimeDelta,omitempty"` // human-readable, e.g. "5d", "12h"
Hint string `json:"hint,omitempty"` // late_updates_skip_likely | sync_lag_or_event_miss
}
type entryRecord struct {
Size uint64 `json:"size"`
Mtime int64 `json:"mtime"`
ETag string `json:"etag,omitempty"`
}
type summaryRecord struct {
Type string `json:"type"`
Directories int64 `json:"directories"`
Files int64 `json:"files"`
SkippedRecent int64 `json:"skippedRecent"`
Missing int64 `json:"missing"`
SizeMismatch int64 `json:"sizeMismatch"`
ETagMismatch int64 `json:"etagMismatch"`
ChunkReorder int64 `json:"chunkReorder"`
OnlyInB int64 `json:"onlyInB"`
TotalErrors int64 `json:"totalErrors"`
}
// simpleFilerClient implements filer_pb.FilerClient for gRPC connections
type simpleFilerClient struct {
grpcAddress pb.ServerAddress
grpcDialOption grpc.DialOption
}
func (c *simpleFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return pb.WithGrpcClient(context.Background(), streamingMode, 0, func(grpcConnection *grpc.ClientConn) error {
client := filer_pb.NewSeaweedFilerClient(grpcConnection)
return fn(client)
}, c.grpcAddress.ToGrpcAddress(), false, c.grpcDialOption)
}
func (c *simpleFilerClient) AdjustedUrl(location *filer_pb.Location) string {
return location.Url
}
func (c *simpleFilerClient) GetDataCenter() string {
return ""
}
func runVerifySync(filerA, filerB pb.ServerAddress, aPath, bPath string,
isActivePassive bool, modifiedTimeAgo time.Duration,
jsonOutput bool,
grpcDialOptionA, grpcDialOptionB grpc.DialOption) error {
clientA := &simpleFilerClient{grpcAddress: filerA, grpcDialOption: grpcDialOptionA}
clientB := &simpleFilerClient{grpcAddress: filerB, grpcDialOption: grpcDialOptionB}
var cutoffTime time.Time
if modifiedTimeAgo > 0 {
cutoffTime = time.Now().Add(-modifiedTimeAgo)
}
if !jsonOutput {
if !cutoffTime.IsZero() {
fmt.Fprintf(os.Stdout, "Verifying files modified before %v (modifiedTimeAgo=%v)\n",
cutoffTime.Format(time.RFC3339), modifiedTimeAgo)
}
fmt.Fprintf(os.Stdout, "Comparing %s%s => %s%s (isActivePassive=%v)\n\n",
filerA, aPath, filerB, bPath, isActivePassive)
}
result := &VerifyResult{jsonOutput: jsonOutput}
ctx := context.Background()
sem := make(chan struct{}, verifySyncConcurrency)
if err := compareDirectory(ctx, clientA, clientB, aPath, bPath, isActivePassive, cutoffTime, sem, result); err != nil {
return err
}
totalErrors := result.missingCount.Load() + result.sizeMismatch.Load() + result.etagMismatch.Load()
if !isActivePassive {
totalErrors += result.onlyInB.Load()
}
if jsonOutput {
summary := summaryRecord{
Type: "SUMMARY",
Directories: result.dirCount.Load(),
Files: result.fileCount.Load(),
SkippedRecent: result.skippedRecent.Load(),
Missing: result.missingCount.Load(),
SizeMismatch: result.sizeMismatch.Load(),
ETagMismatch: result.etagMismatch.Load(),
ChunkReorder: result.chunkReorder.Load(),
OnlyInB: result.onlyInB.Load(),
TotalErrors: totalErrors,
}
writeJSONLine(result, summary)
} else {
fmt.Fprintf(os.Stdout, "\nSummary:\n")
fmt.Fprintf(os.Stdout, " Directories compared: %d\n", result.dirCount.Load())
fmt.Fprintf(os.Stdout, " Files verified: %d\n", result.fileCount.Load())
if result.skippedRecent.Load() > 0 {
fmt.Fprintf(os.Stdout, " Skipped (too recent): %d\n", result.skippedRecent.Load())
}
fmt.Fprintf(os.Stdout, " Missing in B: %d\n", result.missingCount.Load())
fmt.Fprintf(os.Stdout, " Size mismatch: %d\n", result.sizeMismatch.Load())
fmt.Fprintf(os.Stdout, " ETag mismatch: %d\n", result.etagMismatch.Load())
if n := result.chunkReorder.Load(); n > 0 {
fmt.Fprintf(os.Stdout, " Chunk reorder (equal): %d (content-equal, not an error; use -v=1 to list)\n", n)
}
if !isActivePassive {
fmt.Fprintf(os.Stdout, " Only in B: %d\n", result.onlyInB.Load())
}
fmt.Fprintf(os.Stdout, " Total errors: %d\n", totalErrors)
}
if totalErrors > 0 {
return fmt.Errorf("found %d differences", totalErrors)
}
return nil
}
// dirEntries holds one directory's entries buffered and sorted in Go byte
// order, which the merge in compareDirectory requires (it pairs equal names and
// uses eA.Name < eB.Name as the tie-break). The filer's own list order is not
// trusted: since PR #9824 (4.32) a filer forces byte-lexicographic listing
// regardless of SQL collation, while an older filer whose name column uses a
// locale collation lists case-insensitively. Comparing two such clusters yields
// the same name set in a different order, desyncing the merge into spurious
// MISSING / ONLY_IN_B. Sorting both sides client-side makes the order identical
// regardless of filer version or store backend.
//
// The load runs in a background goroutine so both sides of a compareDirectory
// list concurrently; peek/advance block on `ready` until it completes. Memory
// is O(directory size) per side — acceptable for a one-shot verify CLI.
type dirEntries struct {
ready chan struct{} // closed once entries and err are set
entries []*filer_pb.Entry
idx int
err error
}
// newDirEntries starts loading the whole directory and sorting it by Name, then
// returns immediately. The goroutine exits when listing completes, errors, or
// ctx is cancelled; it always closes `ready` so consumers never block forever.
func newDirEntries(ctx context.Context, client filer_pb.FilerClient, dir string) *dirEntries {
s := &dirEntries{ready: make(chan struct{})}
go func() {
defer close(s.ready)
s.err = filer_pb.ReadDirAllEntries(ctx, client, util.FullPath(dir), "",
func(entry *filer_pb.Entry, isLast bool) error {
select {
case <-ctx.Done():
return ctx.Err()
default:
}
s.entries = append(s.entries, entry)
return nil
})
if s.err == nil {
sort.Slice(s.entries, func(i, j int) bool {
return s.entries[i].Name < s.entries[j].Name
})
}
}()
return s
}
// wait blocks until the directory has finished loading and sorting. Its close
// of `ready` happens-before the return, so entries and err are then safe to read
// without additional synchronisation.
func (s *dirEntries) wait() { <-s.ready }
// peek returns the next entry without consuming it, or nil at end-of-stream.
// It blocks until the directory has finished loading and sorting.
func (s *dirEntries) peek() *filer_pb.Entry {
s.wait()
if s.idx >= len(s.entries) {
return nil
}
return s.entries[s.idx]
}
// advance consumes and returns the next entry.
func (s *dirEntries) advance() *filer_pb.Entry {
e := s.peek()
if e != nil {
s.idx++
}
return e
}
func compareDirectory(ctx context.Context,
clientA, clientB filer_pb.FilerClient,
dirA, dirB string,
isActivePassive bool,
cutoffTime time.Time,
sem chan struct{},
result *VerifyResult) error {
// Hold a slot only for this directory's I/O phase (listings + merge).
// Released before recursing so parents never block waiting for children
// to acquire slots — see verifySyncConcurrency for the rationale.
sem <- struct{}{}
released := false
releaseSlot := func() {
if !released {
released = true
<-sem
}
}
defer releaseSlot()
result.dirCount.Add(1)
// A child context cancels the load goroutines (and their listing RPCs) if
// compareDirectory returns early, e.g. on error. Both sides load concurrently.
mergeCtx, cancelMerge := context.WithCancel(ctx)
defer cancelMerge()
entriesA := newDirEntries(mergeCtx, clientA, dirA)
entriesB := newDirEntries(mergeCtx, clientB, dirB)
// Both sides are fully buffered before merging; surface any listing error
// here so a failed or cancelled listing can't emit bogus diffs from partial
// data. wait() makes entries/err visible without extra synchronisation.
entriesA.wait()
entriesB.wait()
if err := entriesA.err; err != nil && err != context.Canceled {
return fmt.Errorf("list %s on filer A: %v", dirA, err)
}
if err := entriesB.err; err != nil && err != context.Canceled {
return fmt.Errorf("list %s on filer B: %v", dirB, err)
}
// collect subdirectories for recursive comparison
type dirPair struct{ a, b string }
var subDirs []dirPair
for entriesA.peek() != nil || entriesB.peek() != nil {
eA := entriesA.peek()
eB := entriesB.peek()
switch {
case eA != nil && (eB == nil || eA.Name < eB.Name):
// entry only in A
entryA := entriesA.advance()
if entryA.IsDirectory {
// Always recurse for missing-in-B directories: a recent
// child write can bump the parent's mtime even though
// older missing files exist underneath. The cutoff is
// applied per-file inside countMissingRecursive.
reportDiff(diffMissing, dirA, entryA, nil, result)
countMissingRecursive(ctx, clientA, path.Join(dirA, entryA.Name), cutoffTime, result)
} else if isTooRecent(entryA, cutoffTime) {
result.skippedRecent.Add(1)
} else {
reportDiff(diffMissing, dirA, entryA, nil, result)
}
case eB != nil && (eA == nil || eB.Name < eA.Name):
// entry only in B
entryB := entriesB.advance()
if !isActivePassive {
if isTooRecent(entryB, cutoffTime) {
result.skippedRecent.Add(1)
} else {
reportDiff(diffOnlyInB, dirB, entryB, nil, result)
}
}
default:
// same name in both
entryA := entriesA.advance()
entryB := entriesB.advance()
if entryA.IsDirectory && entryB.IsDirectory {
subDirs = append(subDirs, dirPair{
a: path.Join(dirA, entryA.Name),
b: path.Join(dirB, entryB.Name),
})
} else if !entryA.IsDirectory && !entryB.IsDirectory {
// Skip if either side was modified recently (sync-lag tolerance).
if isTooRecent(entryA, cutoffTime) || isTooRecent(entryB, cutoffTime) {
result.skippedRecent.Add(1)
} else {
compareEntries(dirA, entryA, entryB, result)
}
} else {
// type mismatch: one is dir, other is file
reportDiff(diffMissing, dirA, entryA, nil, result)
if !isActivePassive {
reportDiff(diffOnlyInB, dirB, entryB, nil, result)
}
}
}
}
// Release our slot before recursing so children can acquire it. Holding
// it across wg.Wait would deadlock once depth exceeds verifySyncConcurrency.
releaseSlot()
if len(subDirs) > 0 {
// Bounded worker pool: cap goroutines per directory level instead
// of spawning one per child. A directory with thousands of subdirs
// would otherwise park ~2KB per waiting goroutine even though
// only `verifySyncConcurrency` can do I/O at once.
workers := verifySyncConcurrency
if len(subDirs) < workers {
workers = len(subDirs)
}
jobs := make(chan dirPair, len(subDirs))
errCh := make(chan error, 1) // first error wins; others dropped
var wg sync.WaitGroup
for i := 0; i < workers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for pair := range jobs {
if err := compareDirectory(ctx, clientA, clientB, pair.a, pair.b, isActivePassive, cutoffTime, sem, result); err != nil {
select {
case errCh <- err:
default:
}
}
}
}()
}
for _, pair := range subDirs {
jobs <- pair
}
close(jobs)
wg.Wait()
close(errCh)
if err := <-errCh; err != nil {
return err
}
}
return nil
}
func compareEntries(dir string, entryA, entryB *filer_pb.Entry, result *VerifyResult) {
result.fileCount.Add(1)
sizeA := filer.FileSize(entryA)
sizeB := filer.FileSize(entryB)
if sizeA != sizeB {
reportDiff(diffSizeMismatch, dir, entryA, entryB, result)
return
}
etagA := filer.ETag(entryA)
etagB := filer.ETag(entryB)
if etagA != etagB {
if isChunkReorder(entryA, entryB) {
// Same bytes, just a different chunk slice order — not a real diff.
reportDiff(diffChunkReorder, dir, entryA, entryB, result)
} else {
reportDiff(diffETagMismatch, dir, entryA, entryB, result)
}
return
}
}
// isChunkReorder reports whether two entries with differing file ETags are
// byte-identical, differing only in chunk slice order. filer.ETagChunks hashes
// per-chunk MD5s in stored order without sorting by offset, so the same bytes
// written by different paths (S3 multipart vs filer.backup) can hash
// differently. If both chunk lists sorted by offset match element-wise on
// (offset, size, ETag), the files cover the same bytes with the same content
// and only the slice order differed.
//
// Applies only when both ETags come from chunks (a stored attr.Md5 is
// order-independent, so a mismatch there is a real content difference) and the
// chunks form a manifest-free, non-overlapping set whose bytes are fully
// determined by the list — overlaps would need timestamp-based visible-interval
// resolution, so those stay ETAG_MISMATCH.
func isChunkReorder(entryA, entryB *filer_pb.Entry) bool {
if !usesChunkETag(entryA) || !usesChunkETag(entryB) {
return false
}
a := sortedSimpleChunks(entryA.GetChunks())
b := sortedSimpleChunks(entryB.GetChunks())
// nil means overlapping or manifest chunks we do not fast-path; a single
// chunk hashes directly and a differing count is a real divergence.
if a == nil || b == nil || len(a) != len(b) || len(a) < 2 {
return false
}
for i := range a {
if a[i].Offset != b[i].Offset || a[i].Size != b[i].Size || a[i].ETag != b[i].ETag {
return false
}
}
return true
}
// usesChunkETag reports whether filer.ETag falls back to ETagChunks for entry,
// i.e. no stored attr.Md5 whole-content hash is available. Mirrors filer.ETag's
// own condition so it agrees with how the compared ETag was computed.
func usesChunkETag(entry *filer_pb.Entry) bool {
return entry.Attributes == nil || entry.Attributes.Md5 == nil
}
// sortedSimpleChunks returns chunks sorted by (offset, ETag) if they are a
// manifest-free, non-overlapping set whose visible bytes are fully determined
// by the list and every chunk carries a usable per-chunk ETag; otherwise nil.
// Overlapping ranges would require resolving the visible interval by chunk
// timestamp, which this fast path does not attempt.
func sortedSimpleChunks(chunks []*filer_pb.FileChunk) []*filer_pb.FileChunk {
sorted := make([]*filer_pb.FileChunk, len(chunks))
copy(sorted, chunks)
sort.Slice(sorted, func(i, j int) bool {
if sorted[i].Offset != sorted[j].Offset {
return sorted[i].Offset < sorted[j].Offset
}
return sorted[i].ETag < sorted[j].ETag
})
prevEnd := int64(0)
for i, c := range sorted {
if c.IsChunkManifest {
return nil
}
// An empty or undecodable ETag is not a content fingerprint, so
// element-wise (offset, size, ETag) equality cannot prove the bytes
// match — decline rather than assert a false content-equality.
if len(util.Base64Md5ToBytes(c.ETag)) == 0 {
return nil
}
if i > 0 && c.Offset < prevEnd {
return nil
}
prevEnd = c.Offset + int64(c.Size)
}
return sorted
}
// mtimeRelation classifies B.mtime vs A.mtime. Both entries must be non-nil.
// Returns relation, absolute delta in seconds, and human-readable string.
func mtimeRelation(entryA, entryB *filer_pb.Entry) (relation, deltaStr string, deltaSec int64) {
if entryA == nil || entryB == nil || entryA.Attributes == nil || entryB.Attributes == nil {
return "", "", 0
}
delta := entryB.Attributes.Mtime - entryA.Attributes.Mtime
abs := delta
if abs < 0 {
abs = -abs
}
switch {
case delta == 0:
return "EQUAL", "0s", 0
case delta > 0:
return "B_NEWER", formatSeconds(abs), abs
default:
return "A_NEWER", formatSeconds(abs), abs
}
}
func formatSeconds(s int64) string {
switch {
case s < 60:
return fmt.Sprintf("%ds", s)
case s < 3600:
return fmt.Sprintf("%dm", s/60)
case s < 86400:
return fmt.Sprintf("%dh", s/3600)
default:
return fmt.Sprintf("%dd", s/86400)
}
}
// hintFor returns an automatic interpretation hint based on mtime relation.
// Only emitted for SIZE_MISMATCH and ETAG_MISMATCH where both entries exist.
func hintFor(relation string) string {
switch relation {
case "B_NEWER":
return "late_updates_skip_likely"
case "A_NEWER":
return "sync_lag_or_event_miss"
}
return ""
}
func reportDiff(diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry, result *VerifyResult) {
switch diffType {
case diffMissing:
result.missingCount.Add(1)
case diffOnlyInB:
result.onlyInB.Add(1)
case diffSizeMismatch:
result.sizeMismatch.Add(1)
case diffETagMismatch:
result.etagMismatch.Add(1)
case diffChunkReorder:
result.chunkReorder.Add(1)
}
if result.jsonOutput {
writeJSONDiff(result, diffType, dir, entryA, entryB)
} else {
// A chunk reorder is content-equal, not a real diff: keep it out of the
// default text report and list it only at -v=1. JSON always emits it so
// the per-record stream matches the summary count.
if diffType == diffChunkReorder && !glog.V(1) {
return
}
writeTextDiff(result, diffType, dir, entryA, entryB)
}
}
func writeTextDiff(result *VerifyResult, diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry) {
entryPath := path.Join(dir, entryA.Name)
result.outputMu.Lock()
defer result.outputMu.Unlock()
switch diffType {
case diffMissing:
if entryA.IsDirectory {
fmt.Fprintf(os.Stdout, "[MISSING] %s/ (directory)\n", entryPath)
} else {
fmt.Fprintf(os.Stdout, "[MISSING] %s (size=%d, etag=%s)\n",
entryPath, filer.FileSize(entryA), filer.ETag(entryA))
}
case diffOnlyInB:
fmt.Fprintf(os.Stdout, "[ONLY_IN_B] %s\n", entryPath)
case diffSizeMismatch:
ann := annotation(entryA, entryB)
fmt.Fprintf(os.Stdout, "[SIZE_MISMATCH] %s (a=%d, b=%d%s)\n",
entryPath, filer.FileSize(entryA), filer.FileSize(entryB), ann)
case diffETagMismatch:
ann := annotation(entryA, entryB)
fmt.Fprintf(os.Stdout, "[ETAG_MISMATCH] %s (a=%s, b=%s%s)\n",
entryPath, filer.ETag(entryA), filer.ETag(entryB), ann)
case diffChunkReorder:
fmt.Fprintf(os.Stdout, "[CHUNK_REORDER] %s (a=%s, b=%s, content-equal: chunks differ only in slice order)\n",
entryPath, filer.ETag(entryA), filer.ETag(entryB))
}
}
// annotation builds the trailing ", mtime: ... [hint]" segment for text output.
// Returns empty string if entries are unavailable.
func annotation(entryA, entryB *filer_pb.Entry) string {
relation, delta, _ := mtimeRelation(entryA, entryB)
if relation == "" {
return ""
}
switch relation {
case "EQUAL":
return ", mtime equal [chunk-level issue]"
case "B_NEWER":
return fmt.Sprintf(", B newer +%s [late-updates skip likely]", delta)
case "A_NEWER":
return fmt.Sprintf(", A newer +%s [sync lag or event miss]", delta)
}
return ""
}
func writeJSONDiff(result *VerifyResult, diffType verifyDiffType, dir string, entryA, entryB *filer_pb.Entry) {
rec := diffRecord{Path: path.Join(dir, entryA.Name)}
switch diffType {
case diffMissing:
rec.Type = "MISSING"
rec.IsDirectory = entryA.IsDirectory
rec.A = toEntryRecord(entryA)
case diffOnlyInB:
rec.Type = "ONLY_IN_B"
// for diffOnlyInB the existing convention passes the entry as entryA
rec.IsDirectory = entryA.IsDirectory
rec.B = toEntryRecord(entryA)
case diffSizeMismatch:
rec.Type = "SIZE_MISMATCH"
rec.A = toEntryRecord(entryA)
rec.B = toEntryRecord(entryB)
relation, delta, _ := mtimeRelation(entryA, entryB)
rec.MtimeRelation = relation
rec.MtimeDelta = delta
rec.Hint = hintFor(relation)
case diffETagMismatch:
rec.Type = "ETAG_MISMATCH"
rec.A = toEntryRecord(entryA)
rec.B = toEntryRecord(entryB)
relation, delta, _ := mtimeRelation(entryA, entryB)
rec.MtimeRelation = relation
rec.MtimeDelta = delta
rec.Hint = hintFor(relation)
case diffChunkReorder:
rec.Type = "CHUNK_REORDER"
rec.A = toEntryRecord(entryA)
rec.B = toEntryRecord(entryB)
relation, delta, _ := mtimeRelation(entryA, entryB)
rec.MtimeRelation = relation
rec.MtimeDelta = delta
// no hint: content-equal, no action needed
}
writeJSONLine(result, rec)
}
func toEntryRecord(entry *filer_pb.Entry) *entryRecord {
if entry == nil {
return nil
}
r := &entryRecord{
Size: filer.FileSize(entry),
ETag: filer.ETag(entry),
}
if entry.Attributes != nil {
r.Mtime = entry.Attributes.Mtime
}
return r
}
// writeJSONLine emits a single JSON object followed by newline. Holds outputMu
// across marshal+write so concurrent goroutines never interleave.
func writeJSONLine(result *VerifyResult, v any) {
data, err := json.Marshal(v)
if err != nil {
glog.Warningf("marshal verify record: %v", err)
return
}
result.outputMu.Lock()
defer result.outputMu.Unlock()
os.Stdout.Write(data)
os.Stdout.Write([]byte{'\n'})
}
func countMissingRecursive(ctx context.Context, client filer_pb.FilerClient, dir string, cutoffTime time.Time, result *VerifyResult) {
err := filer_pb.ReadDirAllEntries(ctx, client, util.FullPath(dir), "",
func(entry *filer_pb.Entry, isLast bool) error {
if entry.IsDirectory {
countMissingRecursive(ctx, client, path.Join(dir, entry.Name), cutoffTime, result)
return nil
}
if !cutoffTime.IsZero() && entry.Attributes != nil && entry.Attributes.Mtime > cutoffTime.Unix() {
result.skippedRecent.Add(1)
return nil
}
reportDiff(diffMissing, dir, entry, nil, result)
return nil
})
if err != nil {
glog.Warningf("list missing directory %s: %v", dir, err)
}
}