Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

19.1 KB
Raw Blame
package shell

import (
	"bytes"
	"context"
	"flag"
	"fmt"
	"io"
	"math"
	"strings"
	"sync"
	"time"

	"slices"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/operation"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/storage"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"go.uber.org/atomic"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func init() {
	Commands = append(Commands, &commandFsVerify{})
}

type commandFsVerify struct {
	env                *CommandEnv
	volumeServers      []pb.ServerAddress
	volumeIds          map[uint32][]pb.ServerAddress
	verbose            *bool
	metadataFromLog    *bool
	pruneEntries       *bool
	concurrency        *int
	modifyTimeAgoAtSec int64
	writer             io.Writer
	waitChan           map[string]chan struct{}
	waitChanLock       sync.RWMutex
}

func (c *commandFsVerify) Name() string {
	return "fs.verify"
}

func (c *commandFsVerify) Help() string {
	return `recursively verify all files under a directory

	fs.verify [-v] [-modifyTimeAgo 1h] [-pruneEntries] [-concurrency 4] /buckets/dir

	-pruneEntries deletes filer entries whose needles are missing from every
	volume location holding the volume (data lost, e.g. after a volume
	truncation). The delete only applies when the entry is unchanged since
	verification, so a re-uploaded file is never pruned.

`
}

func (c *commandFsVerify) HasTag(CommandTag) bool {
	return false
}

func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
	c.env = commandEnv
	c.writer = writer
	fsVerifyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	c.verbose = fsVerifyCommand.Bool("v", false, "print out each processed files")
	modifyTimeAgo := fsVerifyCommand.Duration("modifyTimeAgo", 0, "only include files after this modify time to verify")
	c.concurrency = fsVerifyCommand.Int("concurrency", 0, "number of parallel verification per volume server")
	c.metadataFromLog = fsVerifyCommand.Bool("metadataFromLog", false, "Using  filer log to get metadata")
	c.pruneEntries = fsVerifyCommand.Bool("pruneEntries", false, "delete filer entries whose needles are missing from all volume locations (data lost); a changed entry is never deleted")
	if err = fsVerifyCommand.Parse(args); err != nil {
		return err
	}

	path, parseErr := commandEnv.parseUrl(findInputDirectory(fsVerifyCommand.Args()))
	if parseErr != nil {
		return parseErr
	}

	c.modifyTimeAgoAtSec = int64(modifyTimeAgo.Seconds())
	c.volumeIds = make(map[uint32][]pb.ServerAddress)
	c.waitChan = make(map[string]chan struct{})
	c.volumeServers = []pb.ServerAddress{}
	defer func() {
		c.modifyTimeAgoAtSec = 0
		c.volumeIds = nil
		c.waitChan = nil
		c.volumeServers = nil
	}()

	if err := c.collectVolumeIds(); err != nil {
		return err
	}

	if *c.concurrency > 0 {
		for _, volumeServer := range c.volumeServers {
			volumeServerStr := string(volumeServer)
			c.waitChan[volumeServerStr] = make(chan struct{}, *c.concurrency)
			defer close(c.waitChan[volumeServerStr])
		}
	}
	var fCount, eCount, pCount uint64
	if *c.metadataFromLog {
		var wg sync.WaitGroup
		fCount, eCount, pCount, err = c.verifyProcessMetadata(path, &wg)
		wg.Wait()
	} else {
		fCount, eCount, pCount, err = c.verifyTraverseBfs(path)
	}
	fmt.Fprintf(writer, "verified %d files, error %d files, pruned %d entries \n", fCount, eCount, pCount)
	return err
}

func (c *commandFsVerify) collectVolumeIds() error {
	topologyInfo, _, err := collectTopologyInfo(c.env, 0)
	if err != nil {
		return err
	}
	eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, nodeInfo *master_pb.DataNodeInfo) {
		for _, diskInfo := range nodeInfo.DiskInfos {
			for _, vi := range diskInfo.VolumeInfos {
				volumeServer := pb.NewServerAddressFromDataNode(nodeInfo)
				c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], volumeServer)
				if !slices.Contains(c.volumeServers, volumeServer) {
					c.volumeServers = append(c.volumeServers, volumeServer)
				}
			}
			for _, vi := range diskInfo.EcShardInfos {
				volumeServer := pb.NewServerAddressFromDataNode(nodeInfo)
				c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], volumeServer)
				if !slices.Contains(c.volumeServers, volumeServer) {
					c.volumeServers = append(c.volumeServers, volumeServer)
				}
			}
		}
	})
	return nil
}

func (c *commandFsVerify) verifyChunk(volumeServer pb.ServerAddress, fileId *filer_pb.FileId) error {
	err := operation.WithVolumeServerClient(false, volumeServer, c.env.option.GrpcDialOption,
		func(client volume_server_pb.VolumeServerClient) error {
			_, err := client.VolumeNeedleStatus(context.Background(),
				&volume_server_pb.VolumeNeedleStatusRequest{
					VolumeId: fileId.VolumeId,
					NeedleId: fileId.FileKey})
			return err
		},
	)
	if err != nil && !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) {
		return err
	}
	return nil
}

type ItemEntry struct {
	chunks    []*filer_pb.FileChunk
	rawChunks []*filer_pb.FileChunk
	path      util.FullPath
	attr      *filer_pb.FuseAttributes
}

// resolveAndVerify expands chunk manifests and verifies the entry's needles.
// An unresolvable manifest is an entry-level failure even when raw chunks are
// healthy; raw chunks are still checked so a missing manifest needle can be pruned.
func (c *commandFsVerify) resolveAndVerify(entryPath string, chunks []*filer_pb.FileChunk, errorChunksCount *atomic.Uint64, wg *sync.WaitGroup) (verified bool, hasMissingNeedles bool) {
	dataChunks := chunks
	manifestResolveFailed := false
	if resolved, manifestChunks, resolveErr := filer.ResolveChunkManifest(context.Background(), filer.LookupFn(c.env), chunks, 0, math.MaxInt64, nil); resolveErr == nil {
		dataChunks = append(resolved, manifestChunks...)
	} else {
		manifestResolveFailed = true
		fmt.Fprintf(c.writer, "file: %s failed to resolve chunk manifest (%v), verifying raw chunks\n", entryPath, resolveErr)
	}
	verified, hasMissingNeedles = c.verifyEntry(entryPath, dataChunks, errorChunksCount, wg)
	if manifestResolveFailed && verified {
		// an unreadable manifest is an entry-level failure even with healthy raw needles
		verified = false
		errorChunksCount.CompareAndSwap(0, 1)
	}
	return verified, hasMissingNeedles
}

func (c *commandFsVerify) verifyProcessMetadata(path string, wg *sync.WaitGroup) (fileCount uint64, errCount uint64, prunedCount uint64, err error) {
	processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
		message := resp.EventNotification
		if resp.EventNotification.NewEntry == nil {
			return nil
		}
		chunkCount := len(message.NewEntry.Chunks)
		if chunkCount == 0 {
			return nil
		}
		entryPath := fmt.Sprintf("%s/%s", message.NewParentPath, message.NewEntry.Name)
		errorChunksCount := atomic.NewUint64(0)
		verified, hasMissingNeedles := c.resolveAndVerify(entryPath, message.NewEntry.Chunks, errorChunksCount, wg)
		if !verified {
			if err = c.env.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
				entryResp, errReq := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
					Directory: message.NewParentPath,
					Name:      message.NewEntry.Name,
				})
				if errReq != nil {
					if strings.HasSuffix(errReq.Error(), "no entry is found in filer store") {
						return nil
					}
					return errReq
				}
				if entryResp.Entry.Attributes.Mtime == message.NewEntry.Attributes.Mtime &&
					bytes.Equal(entryResp.Entry.Attributes.Md5, message.NewEntry.Attributes.Md5) {
					fmt.Fprintf(c.writer, "file: %s needles:%d failed:%d\n", entryPath, chunkCount, errorChunksCount.Load())
					errCount++
					if *c.pruneEntries && hasMissingNeedles {
						pruned, pruneErr := c.pruneEntry(
							util.NewFullPath(message.NewParentPath, message.NewEntry.Name),
							message.NewEntry.Attributes,
							message.NewEntry.Chunks)
						if pruneErr != nil {
							fmt.Fprintf(c.writer, "prune %s failed: %v\n", entryPath, pruneErr)
						} else if pruned {
							prunedCount++
						}
					}
				}
				return nil
			}); err != nil {
				return err
			}
			return nil
		}
		if *c.verbose {
			fmt.Fprintf(c.writer, "file: %s needles:%d verified\n", entryPath, chunkCount)
		}
		fileCount++
		return nil
	}
	metadataFollowOption := &pb.MetadataFollowOption{
		ClientName:             "shell_verify",
		ClientId:               util.RandomInt32(),
		ClientEpoch:            0,
		SelfSignature:          0,
		PathPrefix:             path,
		AdditionalPathPrefixes: nil,
		DirectoriesToWatch:     nil,
		StartTsNs:              time.Now().Add(-1 * time.Second * time.Duration(c.modifyTimeAgoAtSec)).UnixNano(),
		StopTsNs:               time.Now().UnixNano(),
		EventErrorType:         pb.DontLogError,
	}
	err = pb.FollowMetadata(c.env.FilerAddress(), c.env.option.GrpcDialOption, metadataFollowOption, processEventFn)
	return fileCount, errCount, prunedCount, err
}

// isNeedleMissingError reports whether a VolumeNeedleStatus error means the
// needle data is lost at that location. It must NOT match "volume not found".
// Newer Go servers and the Rust server return codes.NotFound; older Go
// servers return codes.Unknown with "needle not found <id>" or "EOF" (and EC
// volumes wrap it as "locate in local ec volume: ... needle not found").
// Legacy shapes are matched anchored so unrelated errors don't classify as missing.
func isNeedleMissingError(err error) bool {
	if err == nil {
		return false
	}
	msg := err.Error()
	if st, ok := status.FromError(err); ok {
		switch st.Code() {
		case codes.NotFound:
			return strings.HasPrefix(st.Message(), "needle not found ")
		case codes.Unknown:
			if st.Message() != "" {
				msg = st.Message()
			}
		default:
			return false
		}
	}
	return strings.HasPrefix(msg, "needle not found ") || msg == "EOF" ||
		strings.Contains(msg, "locate in local ec volume:") && strings.HasSuffix(msg, "needle not found")
}

// verifyEntry verifies all chunks of an entry. It returns verified=false when
// any chunk failed. hasMissingNeedles is true when at least one needle is
// missing from every volume location holding its volume, i.e. the data is
// lost rather than a transient or routing error.
func (c *commandFsVerify) verifyEntry(path string, chunks []*filer_pb.FileChunk, errorCount *atomic.Uint64, wg *sync.WaitGroup) (verified bool, hasMissingNeedles bool) {
	fileMsg := fmt.Sprintf("file:%s", path)
	itemIsVerifed := atomic.NewBool(true)
	hasMissingNeedlesFlag := atomic.NewBool(false)
	// tracks this entry's in-flight verifications so the missing-needle
	// decision at the end of each chunk is deterministic
	itemWg := &sync.WaitGroup{}

	for _, chunk := range chunks {
		if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok {
			chunkMissingLocations := atomic.NewUint64(0)
			for _, volumeServer := range volumeIds {
				if *c.concurrency == 0 {
					if err := c.verifyChunk(volumeServer, chunk.Fid); err != nil {
						if isNeedleMissingError(err) {
							chunkMissingLocations.Add(1)
						}
						if !(*c.metadataFromLog && strings.HasSuffix(err.Error(), "not found")) {
							fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v, at volume server %v\n",
								fileMsg, chunk.GetFileIdString(), err, volumeServer)
						}
						if itemIsVerifed.Load() {
							itemIsVerifed.Store(false)
							errorCount.Add(1)
						}
					}
					continue
				}
				c.waitChanLock.RLock()
				waitChan, ok := c.waitChan[string(volumeServer)]
				c.waitChanLock.RUnlock()
				if !ok {
					fmt.Fprintf(c.writer, "%s failed to get channel for %s fileId: %s\n",
						string(volumeServer), fileMsg, chunk.GetFileIdString())
					if itemIsVerifed.Load() {
						itemIsVerifed.Store(false)
						errorCount.Add(1)
					}
					continue
				}
				wg.Add(1)
				itemWg.Add(1)
				waitChan <- struct{}{}
				go func(fChunk *filer_pb.FileChunk, path string, volumeServer pb.ServerAddress, msg string, chunkMissingLocations *atomic.Uint64) {
					defer wg.Done()
					defer itemWg.Done()
					if err := c.verifyChunk(volumeServer, fChunk.Fid); err != nil {
						if isNeedleMissingError(err) {
							chunkMissingLocations.Add(1)
						}
						if !(*c.metadataFromLog && strings.HasSuffix(err.Error(), "not found")) {
							fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v, at volume server %v\n",
								msg, fChunk.GetFileIdString(), err, volumeServer)
						}
						if itemIsVerifed.Load() {
							itemIsVerifed.Store(false)
							errorCount.Add(1)
						}
					}
					<-waitChan
				}(chunk, path, volumeServer, fileMsg, chunkMissingLocations)
			}
			itemWg.Wait()
			if chunkMissingLocations.Load() == uint64(len(volumeIds)) {
				hasMissingNeedlesFlag.Store(true)
			}
		} else {
			if !*c.metadataFromLog {
				err := fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId)
				fmt.Fprintf(c.writer, "%s failed verify fileId %s: %+v\n",
					fileMsg, chunk.GetFileIdString(), err)
			}
			if itemIsVerifed.Load() {
				itemIsVerifed.Store(false)
				errorCount.Add(1)
			}
			break
		}
	}
	return itemIsVerifed.Load(), hasMissingNeedlesFlag.Load()
}

// deleteEntryCutoff splits an entry mtime into the conditional-delete fields:
// seconds keep older filers enforcing the guard, nanoseconds keep same-second
// precision where int64 can hold them. ok is false when neither field can
// express the cutoff and the delete must be skipped rather than sent unguarded.
func deleteEntryCutoff(mtimeSec int64, mtimeNsec int32) (sec, ns int64, ok bool) {
	sec = mtimeSec
	if sec <= 0 {
		return 0, 0, false
	}
	if nsec := int64(mtimeNsec); nsec >= 0 && nsec < 1e9 && sec <= (math.MaxInt64-nsec)/1e9 {
		ns = sec*1e9 + nsec
	}
	return sec, ns, true
}

// pruneEntry deletes an entry whose needles are gone, unless the entry
// changed since verification. Three guards protect a concurrent repair:
// re-lookup with mtime/md5 comparison, an identity check of the chunk set
// (an append or rewrite in the same second as the original mtime changes the
// chunks but not the timestamps), and IfNotModifiedAfter on the delete. The
// entry is only counted as pruned when a follow-up lookup confirms it is gone.
func (c *commandFsVerify) pruneEntry(path util.FullPath, attr *filer_pb.FuseAttributes, expectedChunks []*filer_pb.FileChunk) (pruned bool, err error) {
	dir, name := path.DirAndName()
	expectedIds := chunkFingerprints(expectedChunks)
	lookupErr := c.env.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		lookupResp, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
			Directory: dir,
			Name:      name,
		})
		if err != nil {
			if strings.Contains(err.Error(), "no entry is found in filer store") {
				return nil // already gone
			}
			return err
		}
		att := lookupResp.Entry.GetAttributes()
		if att.GetMtime() != attr.GetMtime() || att.GetMtimeNs() != attr.GetMtimeNs() {
			fmt.Fprintf(c.writer, "skip pruning %s: entry changed since verification\n", path)
			return nil
		}
		if md5 := attr.GetMd5(); len(md5) > 0 && !bytes.Equal(att.GetMd5(), md5) {
			fmt.Fprintf(c.writer, "skip pruning %s: entry changed since verification\n", path)
			return nil
		}
		if !chunksEqual(chunkFingerprints(lookupResp.Entry.GetChunks()), expectedIds) {
			fmt.Fprintf(c.writer, "skip pruning %s: entry changed since verification\n", path)
			return nil
		}
		sec, ns, ok := deleteEntryCutoff(attr.GetMtime(), attr.GetMtimeNs())
		if !ok {
			fmt.Fprintf(c.writer, "skip pruning %s: mtime cannot set a delete condition\n", path)
			return nil
		}
		deleteResp, err := client.DeleteEntry(context.Background(), &filer_pb.DeleteEntryRequest{
			Directory:            dir,
			Name:                 name,
			IfNotModifiedAfter:   sec,
			IfNotModifiedAfterNs: ns,
		})
		if err != nil {
			return err
		}
		if deleteResp.Error != "" {
			return fmt.Errorf("delete entry %s: %s", path, deleteResp.Error)
		}
		// the filer silently skips the delete when the entry advanced past
		// IfNotModifiedAfter between our lookup and the delete; confirm
		confirm, err := client.LookupDirectoryEntry(context.Background(), &filer_pb.LookupDirectoryEntryRequest{
			Directory: dir,
			Name:      name,
		})
		if err != nil {
			if strings.Contains(err.Error(), "no entry is found in filer store") {
				pruned = true
				fmt.Fprintf(c.writer, "pruned entry with missing needles: %s\n", path)
				return nil
			}
			// the delete outcome is unknown — never count it as pruned
			fmt.Fprintf(c.writer, "skip pruning %s: delete not confirmed (%v)\n", path, err)
			return nil
		}
		if confirm.Entry != nil {
			fmt.Fprintf(c.writer, "skip pruning %s: entry changed since verification\n", path)
			return nil
		}
		pruned = true
		fmt.Fprintf(c.writer, "pruned entry with missing needles: %s\n", path)
		return nil
	})
	return pruned, lookupErr
}

// chunkFingerprints maps the file id of every chunk to a set, so two chunk
// lists count as identical when they reference the same needles.
func chunkFingerprints(chunks []*filer_pb.FileChunk) map[string]struct{} {
	ids := make(map[string]struct{}, len(chunks))
	for _, chunk := range chunks {
		if chunk == nil || chunk.Fid == nil {
			continue
		}
		ids[chunk.GetFileIdString()] = struct{}{}
	}
	return ids
}

func chunksEqual(a, b map[string]struct{}) bool {
	if len(a) != len(b) {
		return false
	}
	for id := range a {
		if _, ok := b[id]; !ok {
			return false
		}
	}
	return true
}

func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount uint64, errCount uint64, prunedCount uint64, err error) {
	timeNowAtSec := time.Now().Unix()
	return fileCount, errCount, prunedCount, doTraverseBfsAndSaving(c.env, c.writer, path, false, false,
		func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
			if c.modifyTimeAgoAtSec > 0 {
				if entry.Entry.Attributes != nil && c.modifyTimeAgoAtSec < timeNowAtSec-entry.Entry.Attributes.Mtime {
					return nil
				}
			}
			dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(context.Background(), filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64, nil)
			if resolveErr != nil {
				return fmt.Errorf("failed to ResolveChunkManifest: %+v", resolveErr)
			}
			dataChunks = append(dataChunks, manifestChunks...)
			if len(dataChunks) > 0 {
				select {
				case outputChan <- &ItemEntry{
					chunks:    dataChunks,
					rawChunks: entry.Entry.GetChunks(),
					path:      util.NewFullPath(entry.Dir, entry.Entry.Name),
					attr:      entry.Entry.GetAttributes(),
				}:
				case <-ctx.Done():
					return ctx.Err()
				}
			}
			return nil
		},
		func(outputChan chan interface{}) error {
			var wg sync.WaitGroup
			itemErrCount := atomic.NewUint64(0)
			for itemEntry := range outputChan {
				i := itemEntry.(*ItemEntry)
				itemPath := string(i.path)
				if verified, hasMissingNeedles := c.verifyEntry(itemPath, i.chunks, itemErrCount, &wg); verified {
					if *c.verbose {
						fmt.Fprintf(c.writer, "file: %s needles:%d verified\n", itemPath, len(i.chunks))
					}
					fileCount++
				} else if *c.pruneEntries && hasMissingNeedles {
					pruned, pruneErr := c.pruneEntry(i.path, i.attr, i.rawChunks)
					if pruneErr != nil {
						fmt.Fprintf(c.writer, "prune %s failed: %v\n", itemPath, pruneErr)
					} else if pruned {
						prunedCount++
					}
				}
			}
			wg.Wait()
			errCount = itemErrCount.Load()
			return nil
		})
}