SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
// Package bootstrap is the bucket-level lifecycle walker. The walker
// iterates every entry under a bucket, evaluates every active ActionKey
// against it, and dispatches inline-delete RPCs for currently-due actions;
// not-yet-due entries are left for the meta-log reader to pick up later.
//
// Callback-driven so the listing source and the LifecycleDelete dispatcher
// can be supplied separately (real client or test fake).
package bootstrap
import (
"context"
"strings"
"time"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
"github.com/seaweedfs/seaweedfs/weed/stats"
)
// Entry is the routing-relevant slice of a filer entry. SuccessorModTime
// and NoncurrentIndex are populated only on versioned-bucket walks; the
// retention path bails out conservatively when they're zero / nil.
//
// MPU init directories at .uploads/<id> populate DestKey with the
// destination object key (from entry.Extended[ExtMultipartObjectKey]) so
// rule-prefix matching works on the user's intended path while Path
// stays as the upload directory the dispatcher must rm.
type Entry struct {
Path string
DestKey string
ModTime time.Time
Size int64
IsDirectory bool
IsLatest bool
IsDeleteMarker bool
IsMPUInit bool
NumVersions int
Tags map[string]string
SuccessorModTime time.Time
NoncurrentIndex *int
// VersionID is the S3 version id of this entry, empty for
// non-versioned buckets. Populated by the ListFunc adapter when
// the entry came from a `.versions/` directory; the walker itself
// doesn't use it, but the Dispatcher needs it to address the
// right version on LifecycleDelete.
VersionID string
// Identity is the listed entry's fingerprint; the server refuses the
// delete when the live entry no longer matches it.
Identity *router.EntryIdentity
}
// ListFunc must skip entries with Path <= start so kill-resume picks up
// where the previous run stopped.
type ListFunc func(ctx context.Context, bucket, start string, cb func(*Entry) error) error
// Dispatcher executes one (action, entry) verdict. An error halts the walk;
// the caller decides whether to retry from the recorded last_scanned_path.
type Dispatcher interface {
Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error
}
// Checkpoint is the resume state. Caller persists it under
// /etc/s3/lifecycle/<bucket>/_bootstrap.
type Checkpoint struct {
LastScannedPath string
Completed bool
}
type WalkOptions struct {
Resume string
Now time.Time
}
// Walk iterates entries via list, evaluates each active ActionKey via
// MatchPath + EvaluateAction, and calls Dispatcher.Delete for currently-due
// actions. SCAN_AT_DATE actions are skipped (their bootstrap is scheduled
// separately).
func Walk(ctx context.Context, snap *engine.Snapshot, bucket string, list ListFunc, dispatch Dispatcher, opts WalkOptions) (Checkpoint, error) {
now := opts.Now
if now.IsZero() {
now = time.Now().UTC()
}
cp := Checkpoint{LastScannedPath: opts.Resume}
// Reuse one ObjectInfo across the walk; EvaluateAction reads it
// synchronously without retaining.
var info s3lifecycle.ObjectInfo
err := list(ctx, bucket, opts.Resume, func(entry *Entry) error {
if entry == nil || entry.Path == "" {
return nil
}
// MPU init directories at .uploads/<id> are the one directory
// shape lifecycle cares about; everything else stays out.
if entry.IsDirectory && !entry.IsMPUInit {
cp.LastScannedPath = entry.Path
return nil
}
if err := walkEntry(ctx, snap, bucket, entry, dispatch, now, &info); err != nil {
return err
}
cp.LastScannedPath = entry.Path
return nil
})
if err != nil {
return cp, err
}
cp.Completed = true
return cp, nil
}
func walkEntry(ctx context.Context, snap *engine.Snapshot, bucket string, entry *Entry, dispatch Dispatcher, now time.Time, info *s3lifecycle.ObjectInfo) error {
// MPU init: rule-prefix matching uses the destination key, not the
// .uploads/<id> directory path. A bare directory with no DestKey is
// either a stray dir or an init mid-write before metadata landed —
// skip rather than guess.
matchKey := entry.Path
if entry.IsMPUInit {
if entry.DestKey == "" {
return nil
}
matchKey = entry.DestKey
}
keys := snap.MatchPath(bucket, matchKey, nil)
if len(keys) == 0 {
return nil
}
*info = s3lifecycle.ObjectInfo{
Key: matchKey,
ModTime: entry.ModTime,
Size: entry.Size,
IsLatest: entry.IsLatest,
IsDeleteMarker: entry.IsDeleteMarker,
IsMPUInit: entry.IsMPUInit,
NumVersions: entry.NumVersions,
SuccessorModTime: entry.SuccessorModTime,
NoncurrentIndex: entry.NoncurrentIndex,
Tags: entry.Tags,
}
for _, key := range keys {
action := snap.Action(key)
if action == nil {
continue
}
// DISABLED can be flipped at runtime independent of XML Status,
// so skip it even though EvaluateAction would also reject.
// SCAN_AT_DATE actions are processed here too — the date check
// in EvaluateAction (now.Before(rule.ExpirationDate)) gates the
// dispatch, so pre-date walks are no-ops and post-date walks
// expire eligible objects. The earlier "scan-at-date runs its
// own bootstrap" plan was never wired; until that lands, the
// regular bootstrap walk is the only path that fires
// ExpirationDate rules.
if action.Mode == engine.ModeDisabled {
continue
}
// (kind, info) shape gate: ABORT_MPU only on MPU init records,
// every other kind only on regular objects/versions. Mismatched
// pairs would either dispatch a noncurrent action with empty
// version_id (server BLOCKs, cursor freezes) or dispatch
// ABORT_MPU against a regular object path.
if entry.IsMPUInit && key.ActionKind != s3lifecycle.ActionKindAbortMPU {
continue
}
if !entry.IsMPUInit && key.ActionKind == s3lifecycle.ActionKindAbortMPU {
continue
}
res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, now)
if res.Action == s3lifecycle.ActionNone {
continue
}
if err := dispatch.Delete(ctx, action, entry); err != nil {
glog.Warningf("lifecycle bootstrap: dispatch %s/%s kind=%s: %v",
bucket, entry.Path, key.ActionKind, err)
return err
}
stats.S3LifecycleBootstrapDispatchCounter.WithLabelValues(bucket, key.ActionKind.String()).Inc()
}
return nil
}
// EntryCallback wraps an in-memory slice as a ListFunc; useful for tests.
func EntryCallback(entries []*Entry) ListFunc {
return func(ctx context.Context, bucket, start string, cb func(*Entry) error) error {
for _, e := range entries {
if start != "" && e.Path <= start {
continue
}
if err := cb(e); err != nil {
return err
}
}
return nil
}
}
func HasPrefix(path, prefix string) bool { return strings.HasPrefix(path, prefix) }