SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package command
import (
"context"
"fmt"
"sort"
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)
type lookupEntryFilerClient struct {
entries map[string]*filer_pb.Entry
}
func (c *lookupEntryFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
return fn(&lookupEntrySeaweedClient{entries: c.entries})
}
func (c *lookupEntryFilerClient) AdjustedUrl(location *filer_pb.Location) string {
return location.Url
}
func (c *lookupEntryFilerClient) GetDataCenter() string { return "" }
type lookupEntrySeaweedClient struct {
filer_pb.SeaweedFilerClient
entries map[string]*filer_pb.Entry
}
func (c *lookupEntrySeaweedClient) LookupDirectoryEntry(ctx context.Context, in *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
entry, found := c.entries[in.Directory+"/"+in.Name]
if !found {
return nil, filer_pb.ErrNotFound
}
return &filer_pb.LookupDirectoryEntryResponse{Entry: entry}, nil
}
func (c *lookupEntrySeaweedClient) ListEntries(_ context.Context, in *filer_pb.ListEntriesRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
var entries []*filer_pb.Entry
for fullPath, entry := range c.entries {
if dir, _ := util.FullPath(fullPath).DirAndName(); dir == in.Directory {
entries = append(entries, entry)
}
}
sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name })
if in.StartFromFileName != "" {
idx := sort.Search(len(entries), func(i int) bool { return entries[i].Name >= in.StartFromFileName })
if !in.InclusiveStartFrom {
for idx < len(entries) && entries[idx].Name == in.StartFromFileName {
idx++
}
}
entries = entries[idx:]
}
limit := in.Limit
if limit == 0 {
limit = filer.PaginationSize // the filer substitutes DirListingLimit for 0
}
if uint64(len(entries)) > uint64(limit) {
entries = entries[:limit]
}
return &verifyTestStream{entries: entries}, nil
}
type recordSink struct {
created map[string]*filer_pb.Entry
deleted []string
}
func (s *recordSink) GetName() string { return "record" }
func (s *recordSink) Initialize(configuration util.Configuration, prefix string) error {
return nil
}
func (s *recordSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
s.deleted = append(s.deleted, key)
return nil
}
func (s *recordSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
s.created[key] = entry
return nil
}
func (s *recordSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
return true, nil
}
func (s *recordSink) GetSinkToDirectory() string { return "/" }
func (s *recordSink) GetDestinationIdentity() string { return "record" }
func (s *recordSink) SetSourceFiler(fs *source.FilerSource) {}
func (s *recordSink) IsIncremental() bool { return false }
func newFlattener(entries map[string]*filer_pb.Entry) (*s3VersionFlattener, *recordSink) {
rs := &recordSink{created: make(map[string]*filer_pb.Entry)}
f := &s3VersionFlattener{
filerSource: &lookupEntryFilerClient{entries: entries},
dataSink: rs,
sourcePath: "/buckets",
targetPath: "/",
doDeleteFiles: true,
}
return f, rs
}
func event(dir, parent string, oldEntry, newEntry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
return &filer_pb.SubscribeMetadataResponse{
Directory: dir,
EventNotification: &filer_pb.EventNotification{
OldEntry: oldEntry,
NewEntry: newEntry,
NewParentPath: parent,
},
}
}
func TestFlattenDropsVersionFileEvents(t *testing.T) {
f, rs := newFlattener(map[string]*filer_pb.Entry{})
v := &filer_pb.Entry{Name: "v_abc", Chunks: []*filer_pb.FileChunk{{FileId: "1,01"}}}
for _, resp := range []*filer_pb.SubscribeMetadataResponse{
event("/buckets/b/k.versions", "", nil, v),
event("/buckets/b/k.versions", "", v, nil),
event("/buckets/b/k.versions", "/buckets/b/k.versions", v, v),
} {
handled, err := f.process(resp)
require.NoError(t, err)
require.True(t, handled, "version file events should be consumed")
}
require.Empty(t, rs.created)
require.Empty(t, rs.deleted)
}
func TestFlattenPassesThroughNormalEvents(t *testing.T) {
f, rs := newFlattener(nil)
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, &filer_pb.Entry{Name: "k"}))
require.NoError(t, err)
require.False(t, handled)
// a regular file literally named v_foo outside a .versions dir is not a
// version file
handled, err = f.process(event("/buckets/b", "/buckets/b", nil, &filer_pb.Entry{Name: "v_notes"}))
require.NoError(t, err)
require.False(t, handled)
// a plain file whose name ends with .versions is not a versions directory
handled, err = f.process(event("/buckets/b", "/buckets/b", nil, &filer_pb.Entry{Name: "x.versions"}))
require.NoError(t, err)
require.False(t, handled)
require.Empty(t, rs.created)
}
func TestFlattenUploadsLatestVersion(t *testing.T) {
version := &filer_pb.Entry{
Name: "v_2",
Attributes: &filer_pb.FuseAttributes{Mtime: 7},
Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}},
}
f, rs := newFlattener(map[string]*filer_pb.Entry{"/buckets/b/k.versions/v_2": version})
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
require.NoError(t, err)
require.True(t, handled)
require.Same(t, version, rs.created["/b/k"])
require.Empty(t, rs.deleted)
}
func TestFlattenDeletesOnLatestDeleteMarker(t *testing.T) {
f, rs := newFlattener(nil)
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionIsDeleteMarker: []byte("true")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", dir, dir))
require.NoError(t, err)
require.True(t, handled)
require.Equal(t, []string{"/b/k"}, rs.deleted)
require.Empty(t, rs.created)
}
func TestFlattenDeleteHonorsDoDeleteFiles(t *testing.T) {
f, rs := newFlattener(nil)
f.doDeleteFiles = false
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionIsDeleteMarker: []byte("true")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", dir, dir))
require.NoError(t, err)
require.True(t, handled)
require.Empty(t, rs.deleted)
}
func TestFlattenDeleteOnClearedPointerAndRemovedDir(t *testing.T) {
f, rs := newFlattener(nil)
withPointer := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_1")},
}
cleared := &filer_pb.Entry{Name: "k.versions", IsDirectory: true, Extended: map[string][]byte{}}
handled, err := f.process(event("/buckets/b", "/buckets/b", withPointer, cleared))
require.NoError(t, err)
require.True(t, handled)
require.Equal(t, []string{"/b/k"}, rs.deleted)
rs.deleted = nil
handled, err = f.process(event("/buckets/b", "", cleared, nil))
require.NoError(t, err)
require.True(t, handled)
require.Equal(t, []string{"/b/k"}, rs.deleted)
}
func TestFlattenFreshVersionsDirIsSkipped(t *testing.T) {
f, rs := newFlattener(nil)
// a .versions dir create carries no pointer yet — the update that sets it
// follows in the same write
dir := &filer_pb.Entry{Name: "k.versions", IsDirectory: true}
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
require.NoError(t, err)
require.True(t, handled)
require.Empty(t, rs.deleted)
require.Empty(t, rs.created)
}
func TestFlattenNullVersionIsLatest(t *testing.T) {
nullObject := &filer_pb.Entry{
Name: "k",
Attributes: &filer_pb.FuseAttributes{Mtime: 9},
Chunks: []*filer_pb.FileChunk{{FileId: "1,09"}},
}
f, rs := newFlattener(map[string]*filer_pb.Entry{"/buckets/b/k": nullObject})
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
require.NoError(t, err)
require.True(t, handled)
require.Same(t, nullObject, rs.created["/b/k"])
}
func TestFlattenNullVersionDeletedRescansNewestVersion(t *testing.T) {
// deleting ?versionId=null removes the regular object without touching the
// null-is-latest pointer: the flattened key must fall back to the newest
// remaining version, the same rescan the S3 read path does
older := &filer_pb.Entry{Name: "v_00000000000000010000000000000000", Chunks: []*filer_pb.FileChunk{{FileId: "1,01"}}}
newer := &filer_pb.Entry{Name: "v_00000000000000020000000000000000", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
f, rs := newFlattener(map[string]*filer_pb.Entry{
"/buckets/b/k.versions/" + older.Name: older,
"/buckets/b/k.versions/" + newer.Name: newer,
})
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
require.NoError(t, err)
require.True(t, handled)
require.Same(t, newer, rs.created["/b/k"])
require.Empty(t, rs.deleted)
}
func TestFlattenNullVersionDeletedNewestIsDeleteMarker(t *testing.T) {
marker := &filer_pb.Entry{
Name: "v_00000000000000020000000000000000",
Extended: map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte("true")},
}
older := &filer_pb.Entry{Name: "v_00000000000000010000000000000000", Chunks: []*filer_pb.FileChunk{{FileId: "1,01"}}}
f, rs := newFlattener(map[string]*filer_pb.Entry{
"/buckets/b/k.versions/" + older.Name: older,
"/buckets/b/k.versions/" + marker.Name: marker,
})
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
require.NoError(t, err)
require.True(t, handled)
require.Equal(t, []string{"/b/k"}, rs.deleted)
require.Empty(t, rs.created)
}
func TestFlattenNullVersionDeletedNoVersionsLeftDeletes(t *testing.T) {
f, rs := newFlattener(map[string]*filer_pb.Entry{})
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
require.NoError(t, err)
require.True(t, handled)
require.Equal(t, []string{"/b/k"}, rs.deleted)
}
func TestFlattenNullVersionDeletedScansPastOnePage(t *testing.T) {
// old-format version names sort oldest-first and the filer caps one
// listing at DirListingLimit, so with a long history the newest version
// sits beyond the first page — the rescan must read every page
entries := map[string]*filer_pb.Entry{}
var newest *filer_pb.Entry
for i := 1; i <= filer.PaginationSize+1; i++ {
v := &filer_pb.Entry{
Name: fmt.Sprintf("v_%032x", i),
Chunks: []*filer_pb.FileChunk{{FileId: fmt.Sprintf("1,%02x", i)}},
}
entries["/buckets/b/k.versions/"+v.Name] = v
newest = v
}
f, rs := newFlattener(entries)
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
require.NoError(t, err)
require.True(t, handled)
require.Same(t, newest, rs.created["/b/k"])
}
func TestFlattenVersionFileEventRefreshesCurrentVersion(t *testing.T) {
// a tag-only update touches the version file, not the pointer: re-resolving
// the pointer must still upload the current version to the flat key
version := &filer_pb.Entry{Name: "v_2", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
}
f, rs := newFlattener(map[string]*filer_pb.Entry{
"/buckets/b/k.versions": dir,
"/buckets/b/k.versions/v_2": version,
})
tagUpdate := &filer_pb.Entry{Name: "v_2", Extended: map[string][]byte{"X-Amz-Tagging-env": []byte("prod")}}
handled, err := f.process(event("/buckets/b/k.versions", "/buckets/b/k.versions", tagUpdate, tagUpdate))
require.NoError(t, err)
require.True(t, handled)
require.Same(t, version, rs.created["/b/k"])
}
func TestFlattenExcludedObjectNameIsNotCopiedOrDeleted(t *testing.T) {
f, rs := newFlattener(nil)
f.excludeFileNames = wildcard.CompileWildcardMatchers("*.tmp")
dir := &filer_pb.Entry{
Name: "notes.tmp.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionIsDeleteMarker: []byte("true")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", dir, dir))
require.NoError(t, err)
require.True(t, handled)
require.Empty(t, rs.deleted, "delete on the excluded flat key must be skipped")
handled, err = f.process(event("/buckets/b", "", dir, nil))
require.NoError(t, err)
require.True(t, handled)
require.Empty(t, rs.deleted)
}
func TestFlattenRegularPathFollowsVersionsPointer(t *testing.T) {
// deleting the null object must not remove the flattened key while the
// .versions pointer names a live version
version := &filer_pb.Entry{Name: "v_2", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
}
f, rs := newFlattener(map[string]*filer_pb.Entry{
"/buckets/b/k.versions": dir,
"/buckets/b/k.versions/v_2": version,
})
handled, err := f.process(event("/buckets/b", "", &filer_pb.Entry{Name: "k"}, nil))
require.NoError(t, err)
require.True(t, handled)
require.Same(t, version, rs.created["/b/k"])
require.Empty(t, rs.deleted)
}
func TestFlattenRenameLeavesDeleteForGenericPath(t *testing.T) {
// rename a -> b where only b carries a .versions directory: the flattener
// refreshes b through the pointer but the old flattened key held the moved
// file, so the event must reach the generic path as a lone delete of a
version := &filer_pb.Entry{Name: "v_9", Chunks: []*filer_pb.FileChunk{{FileId: "1,09"}}}
dir := &filer_pb.Entry{
Name: "b.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_9")},
}
f, rs := newFlattener(map[string]*filer_pb.Entry{
"/buckets/b/b.versions": dir,
"/buckets/b/b.versions/v_9": version,
})
resp := event("/buckets/b", "/buckets/b", &filer_pb.Entry{Name: "a"}, &filer_pb.Entry{Name: "b"})
handled, err := f.process(resp)
require.NoError(t, err)
require.False(t, handled, "old side has no .versions sibling — generic path must delete a")
require.NotNil(t, resp.EventNotification.OldEntry)
require.Nil(t, resp.EventNotification.NewEntry)
require.Same(t, version, rs.created["/b/b"])
}
func TestFlattenRenameLeavesCreateForGenericPath(t *testing.T) {
// rename a -> b where only a carries a .versions directory: the pointer
// still decides flattened a, so the generic path must see a lone create
// of b and never delete the flattened old key
version := &filer_pb.Entry{Name: "v_2", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
dir := &filer_pb.Entry{
Name: "a.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
}
f, rs := newFlattener(map[string]*filer_pb.Entry{
"/buckets/b/a.versions": dir,
"/buckets/b/a.versions/v_2": version,
})
resp := event("/buckets/b", "/buckets/b", &filer_pb.Entry{Name: "a"}, &filer_pb.Entry{Name: "b"})
handled, err := f.process(resp)
require.NoError(t, err)
require.False(t, handled)
require.Nil(t, resp.EventNotification.OldEntry)
require.NotNil(t, resp.EventNotification.NewEntry)
require.Same(t, version, rs.created["/b/a"])
require.Empty(t, rs.deleted)
}
func TestFlattenRenameBothSidesVersioned(t *testing.T) {
versionA := &filer_pb.Entry{Name: "v_2", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
versionB := &filer_pb.Entry{Name: "v_9", Chunks: []*filer_pb.FileChunk{{FileId: "1,09"}}}
f, rs := newFlattener(map[string]*filer_pb.Entry{
"/buckets/b/a.versions": {
Name: "a.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
},
"/buckets/b/a.versions/v_2": versionA,
"/buckets/b/b.versions": {
Name: "b.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_9")},
},
"/buckets/b/b.versions/v_9": versionB,
})
resp := event("/buckets/b", "/buckets/b", &filer_pb.Entry{Name: "a"}, &filer_pb.Entry{Name: "b"})
handled, err := f.process(resp)
require.NoError(t, err)
require.True(t, handled)
require.NotNil(t, resp.EventNotification.OldEntry)
require.NotNil(t, resp.EventNotification.NewEntry)
require.Same(t, versionA, rs.created["/b/a"])
require.Same(t, versionB, rs.created["/b/b"])
}
func TestFlattenDeleteMarkerVersionEntry(t *testing.T) {
marker := &filer_pb.Entry{
Name: "v_3",
Extended: map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte("true")},
}
f, rs := newFlattener(map[string]*filer_pb.Entry{"/buckets/b/k.versions/v_3": marker})
dir := &filer_pb.Entry{
Name: "k.versions",
IsDirectory: true,
Extended: map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_3")},
}
handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
require.NoError(t, err)
require.True(t, handled)
require.Equal(t, []string{"/b/k"}, rs.deleted)
}