SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package shell
import (
"context"
"errors"
"flag"
"fmt"
"io"
"strings"
"sync/atomic"
"time"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
s3_constants "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/protobuf/proto"
)
func init() {
Commands = append(Commands, &commandS3VersionsAudit{})
}
type commandS3VersionsAudit struct{}
func (c *commandS3VersionsAudit) Name() string {
return "s3.versions.audit"
}
func (c *commandS3VersionsAudit) Help() string {
return `audit .versions/ directories under a prefix for stranded pointer/missing-file state
Walks every entry under the given prefix and, for each directory whose
name ends in ".versions", checks whether its extended-attr latest-version
pointer references a file that actually exists in the directory.
Reports counts for:
- directories scanned
- clean (no pointer, or pointer matches existing file)
- stranded (pointer set but file is missing) — the symptom seen by
Veeam/etc. as "Storage not found" on the next GET
- orphan (directory has files lacking the version-id extended attr,
which the post-delete cleanup path will refuse to rm)
- empty (no pointer and no children — residue of a fully-drained key;
every GET of the key replays the read-side self-heal rescan)
Example:
# Audit a whole bucket
s3.versions.audit -prefix /buckets/mybucket
# Audit a specific client subtree, print each finding
s3.versions.audit -prefix /buckets/mybucket/Veeam/Backup/groupsoftware/Clients/<uuid>/ -v
# Dry run (default) — read-only, prints what would be healed
# Add -heal to clear stranded pointers in place (calls the same path
# the read-side self-heal uses)
s3.versions.audit -prefix /buckets/mybucket -heal
This command is read-only by default. With -heal, it clears the stale
latest-version pointer on stranded directories and removes empty ones;
the blob is already gone, so reads then return NoSuchKey via the
clean-miss path instead of replaying the 10-retry self-heal loop on
every request.
`
}
func (c *commandS3VersionsAudit) HasTag(CommandTag) bool {
return false
}
func (c *commandS3VersionsAudit) Do(args []string, commandEnv *CommandEnv, writer io.Writer) error {
cmd := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
prefix := cmd.String("prefix", "", "filer path to audit recursively (e.g. /buckets/mybucket)")
verbose := cmd.Bool("v", false, "print each stranded/orphan directory as it's found")
doHeal := cmd.Bool("heal", false, "clear the latest-version pointer on stranded directories (default: read-only)")
if err := cmd.Parse(args); err != nil {
return err
}
if *prefix == "" {
return fmt.Errorf("-prefix is required")
}
// Counters
var (
dirsScanned uint64
versionsDirs uint64
clean uint64
stranded uint64
orphanOnly uint64
empty uint64
healed uint64
healFailed uint64
)
start := time.Now()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
return filer_pb.TraverseBfs(ctx, &filerClientWrapper{client: client}, util.FullPath(*prefix), func(parentPath util.FullPath, entry *filer_pb.Entry) error {
atomic.AddUint64(&dirsScanned, 1)
if !entry.IsDirectory {
return nil
}
if !strings.HasSuffix(entry.Name, ".versions") {
return nil
}
atomic.AddUint64(&versionsDirs, 1)
// What does the pointer name?
var pointerFile string
if entry.Extended != nil {
if v, ok := entry.Extended[s3_constants.ExtLatestVersionFileNameKey]; ok {
pointerFile = string(v)
}
}
// List the children to see if the pointer's file exists and to
// count entries without an ExtVersionIdKey (orphans that block
// non-recursive teardown). filer_pb.List returns one 1024-entry
// page; walk all pages so a large .versions/ directory doesn't
// produce a false positive "stranded" report from only seeing
// the first page.
versionsPath := string(parentPath) + "/" + entry.Name
pointerSeen := false
hasOrphan := false
hasChild := false
const auditPageSize = 1024
var startName string
for {
pageEntries := 0
var lastEntryName string
lookupErr := filer_pb.List(ctx, &filerClientWrapper{client: client}, versionsPath, "", func(child *filer_pb.Entry, isLast bool) error {
if child == nil {
return nil
}
pageEntries++
lastEntryName = child.Name
hasChild = true
hasVersionId := false
if child.Extended != nil {
if _, ok := child.Extended[s3_constants.ExtVersionIdKey]; ok {
hasVersionId = true
}
}
if pointerFile != "" && child.Name == pointerFile {
pointerSeen = true
}
if !hasVersionId {
hasOrphan = true
}
return nil
}, startName, startName != "", auditPageSize)
if lookupErr != nil {
fmt.Fprintf(writer, "list %s: %v\n", versionsPath, lookupErr)
return nil
}
if pageEntries < auditPageSize {
break
}
startName = lastEntryName
}
switch {
case pointerFile == "":
// No pointer set. Empty, orphan-only and clean are mutually
// exclusive so the final report's category counts sum to
// versionsDirs.
switch {
case !hasChild:
atomic.AddUint64(&empty, 1)
if *verbose {
fmt.Fprintf(writer, "empty: %s\n", versionsPath)
}
if *doHeal {
if err := removeEmptyVersionsDir(ctx, client, parentPath, entry); err != nil {
atomic.AddUint64(&healFailed, 1)
fmt.Fprintf(writer, "heal failed: %s: %v\n", versionsPath, err)
} else {
atomic.AddUint64(&healed, 1)
}
}
case hasOrphan:
atomic.AddUint64(&orphanOnly, 1)
default:
atomic.AddUint64(&clean, 1)
}
case pointerSeen:
atomic.AddUint64(&clean, 1)
default:
// Pointer names a file that the listing does NOT contain.
atomic.AddUint64(&stranded, 1)
if *verbose {
fmt.Fprintf(writer, "stranded: %s pointer=%s orphan=%v\n", versionsPath, pointerFile, hasOrphan)
}
if *doHeal {
if err := healStrandedPointer(ctx, client, parentPath, entry); err != nil {
atomic.AddUint64(&healFailed, 1)
fmt.Fprintf(writer, "heal failed: %s: %v\n", versionsPath, err)
} else {
atomic.AddUint64(&healed, 1)
}
}
}
return nil
})
})
elapsed := time.Since(start)
fmt.Fprintf(writer, "audit complete in %s\n", elapsed)
fmt.Fprintf(writer, " total entries scanned : %d\n", dirsScanned)
fmt.Fprintf(writer, " .versions/ directories: %d\n", versionsDirs)
fmt.Fprintf(writer, " clean : %d\n", clean)
fmt.Fprintf(writer, " stranded : %d\n", stranded)
fmt.Fprintf(writer, " orphan-only : %d\n", orphanOnly)
fmt.Fprintf(writer, " empty : %d\n", empty)
if *doHeal {
fmt.Fprintf(writer, " healed : %d\n", healed)
fmt.Fprintf(writer, " heal failed : %d\n", healFailed)
}
return err
}
// healStrandedPointer clears the latest-version pointer extended attrs
// on a stranded .versions/ directory. The blob the pointer names is
// already gone; clearing the pointer makes subsequent reads return
// NoSuchKey via the clean-miss path instead of replaying the read-side
// self-heal on every request.
func healStrandedPointer(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath util.FullPath, entry *filer_pb.Entry) error {
if entry.Extended == nil {
return nil
}
readEntry := proto.Clone(entry).(*filer_pb.Entry)
delete(entry.Extended, s3_constants.ExtLatestVersionIdKey)
delete(entry.Extended, s3_constants.ExtLatestVersionFileNameKey)
// Also clear the cached list metadata so a stale size/mtime/etag
// can't be served back; those will be repopulated on the next PUT.
delete(entry.Extended, s3_constants.ExtLatestVersionSizeKey)
delete(entry.Extended, s3_constants.ExtLatestVersionMtimeKey)
delete(entry.Extended, s3_constants.ExtLatestVersionETagKey)
delete(entry.Extended, s3_constants.ExtLatestVersionOwnerKey)
delete(entry.Extended, s3_constants.ExtLatestVersionIsDeleteMarker)
// A version written since the audit read the directory has moved the
// pointer, and clearing it would hide that version.
_, err := updateEntryIfUnchanged(ctx, client, string(parentPath), entry, readEntry)
return err
}
// removeEmptyVersionsDir deletes a .versions/ directory that holds no
// children: residue of a fully-drained key. Non-recursive on purpose so a
// concurrent write landing a new version fails the delete instead of being
// lost with it.
func removeEmptyVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, parentPath util.FullPath, entry *filer_pb.Entry) error {
resp, err := client.DeleteEntry(ctx, &filer_pb.DeleteEntryRequest{
Directory: string(parentPath),
Name: entry.Name,
IsDeleteData: true,
})
if err != nil {
return err
}
if resp.Error != "" {
return errors.New(resp.Error)
}
return nil
}
// filerClientWrapper adapts a raw SeaweedFilerClient to the
// filer_pb.FilerClient interface that List / TraverseBfs expect.
type filerClientWrapper struct {
client filer_pb.SeaweedFilerClient
}
func (w *filerClientWrapper) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return fn(w.client)
}
func (w *filerClientWrapper) AdjustedUrl(location *filer_pb.Location) string {
return location.Url
}
func (w *filerClientWrapper) GetDataCenter() string {
return ""
}