Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

18.0 KB
Raw Blame
package command

import (
	"context"
	"fmt"
	"sort"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/replication/source"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
	"github.com/stretchr/testify/require"
	"google.golang.org/grpc"
)

type lookupEntryFilerClient struct {
	entries map[string]*filer_pb.Entry
}

func (c *lookupEntryFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
	return fn(&lookupEntrySeaweedClient{entries: c.entries})
}
func (c *lookupEntryFilerClient) AdjustedUrl(location *filer_pb.Location) string {
	return location.Url
}
func (c *lookupEntryFilerClient) GetDataCenter() string { return "" }

type lookupEntrySeaweedClient struct {
	filer_pb.SeaweedFilerClient
	entries map[string]*filer_pb.Entry
}

func (c *lookupEntrySeaweedClient) LookupDirectoryEntry(ctx context.Context, in *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) {
	entry, found := c.entries[in.Directory+"/"+in.Name]
	if !found {
		return nil, filer_pb.ErrNotFound
	}
	return &filer_pb.LookupDirectoryEntryResponse{Entry: entry}, nil
}

func (c *lookupEntrySeaweedClient) ListEntries(_ context.Context, in *filer_pb.ListEntriesRequest, _ ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
	var entries []*filer_pb.Entry
	for fullPath, entry := range c.entries {
		if dir, _ := util.FullPath(fullPath).DirAndName(); dir == in.Directory {
			entries = append(entries, entry)
		}
	}
	sort.Slice(entries, func(i, j int) bool { return entries[i].Name < entries[j].Name })
	if in.StartFromFileName != "" {
		idx := sort.Search(len(entries), func(i int) bool { return entries[i].Name >= in.StartFromFileName })
		if !in.InclusiveStartFrom {
			for idx < len(entries) && entries[idx].Name == in.StartFromFileName {
				idx++
			}
		}
		entries = entries[idx:]
	}
	limit := in.Limit
	if limit == 0 {
		limit = filer.PaginationSize // the filer substitutes DirListingLimit for 0
	}
	if uint64(len(entries)) > uint64(limit) {
		entries = entries[:limit]
	}
	return &verifyTestStream{entries: entries}, nil
}

type recordSink struct {
	created map[string]*filer_pb.Entry
	deleted []string
}

func (s *recordSink) GetName() string { return "record" }
func (s *recordSink) Initialize(configuration util.Configuration, prefix string) error {
	return nil
}
func (s *recordSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
	s.deleted = append(s.deleted, key)
	return nil
}
func (s *recordSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
	s.created[key] = entry
	return nil
}
func (s *recordSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
	return true, nil
}
func (s *recordSink) GetSinkToDirectory() string            { return "/" }
func (s *recordSink) GetDestinationIdentity() string        { return "record" }
func (s *recordSink) SetSourceFiler(fs *source.FilerSource) {}
func (s *recordSink) IsIncremental() bool                   { return false }

func newFlattener(entries map[string]*filer_pb.Entry) (*s3VersionFlattener, *recordSink) {
	rs := &recordSink{created: make(map[string]*filer_pb.Entry)}
	f := &s3VersionFlattener{
		filerSource:   &lookupEntryFilerClient{entries: entries},
		dataSink:      rs,
		sourcePath:    "/buckets",
		targetPath:    "/",
		doDeleteFiles: true,
	}
	return f, rs
}

func event(dir, parent string, oldEntry, newEntry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse {
	return &filer_pb.SubscribeMetadataResponse{
		Directory: dir,
		EventNotification: &filer_pb.EventNotification{
			OldEntry:      oldEntry,
			NewEntry:      newEntry,
			NewParentPath: parent,
		},
	}
}

func TestFlattenDropsVersionFileEvents(t *testing.T) {
	f, rs := newFlattener(map[string]*filer_pb.Entry{})
	v := &filer_pb.Entry{Name: "v_abc", Chunks: []*filer_pb.FileChunk{{FileId: "1,01"}}}

	for _, resp := range []*filer_pb.SubscribeMetadataResponse{
		event("/buckets/b/k.versions", "", nil, v),
		event("/buckets/b/k.versions", "", v, nil),
		event("/buckets/b/k.versions", "/buckets/b/k.versions", v, v),
	} {
		handled, err := f.process(resp)
		require.NoError(t, err)
		require.True(t, handled, "version file events should be consumed")
	}
	require.Empty(t, rs.created)
	require.Empty(t, rs.deleted)
}

func TestFlattenPassesThroughNormalEvents(t *testing.T) {
	f, rs := newFlattener(nil)

	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, &filer_pb.Entry{Name: "k"}))
	require.NoError(t, err)
	require.False(t, handled)

	// a regular file literally named v_foo outside a .versions dir is not a
	// version file
	handled, err = f.process(event("/buckets/b", "/buckets/b", nil, &filer_pb.Entry{Name: "v_notes"}))
	require.NoError(t, err)
	require.False(t, handled)

	// a plain file whose name ends with .versions is not a versions directory
	handled, err = f.process(event("/buckets/b", "/buckets/b", nil, &filer_pb.Entry{Name: "x.versions"}))
	require.NoError(t, err)
	require.False(t, handled)
	require.Empty(t, rs.created)
}

func TestFlattenUploadsLatestVersion(t *testing.T) {
	version := &filer_pb.Entry{
		Name:       "v_2",
		Attributes: &filer_pb.FuseAttributes{Mtime: 7},
		Chunks:     []*filer_pb.FileChunk{{FileId: "1,02"}},
	}
	f, rs := newFlattener(map[string]*filer_pb.Entry{"/buckets/b/k.versions/v_2": version})

	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Same(t, version, rs.created["/b/k"])
	require.Empty(t, rs.deleted)
}

func TestFlattenDeletesOnLatestDeleteMarker(t *testing.T) {
	f, rs := newFlattener(nil)
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionIsDeleteMarker: []byte("true")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", dir, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Equal(t, []string{"/b/k"}, rs.deleted)
	require.Empty(t, rs.created)
}

func TestFlattenDeleteHonorsDoDeleteFiles(t *testing.T) {
	f, rs := newFlattener(nil)
	f.doDeleteFiles = false
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionIsDeleteMarker: []byte("true")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", dir, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Empty(t, rs.deleted)
}

func TestFlattenDeleteOnClearedPointerAndRemovedDir(t *testing.T) {
	f, rs := newFlattener(nil)
	withPointer := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_1")},
	}
	cleared := &filer_pb.Entry{Name: "k.versions", IsDirectory: true, Extended: map[string][]byte{}}

	handled, err := f.process(event("/buckets/b", "/buckets/b", withPointer, cleared))
	require.NoError(t, err)
	require.True(t, handled)
	require.Equal(t, []string{"/b/k"}, rs.deleted)

	rs.deleted = nil
	handled, err = f.process(event("/buckets/b", "", cleared, nil))
	require.NoError(t, err)
	require.True(t, handled)
	require.Equal(t, []string{"/b/k"}, rs.deleted)
}

func TestFlattenFreshVersionsDirIsSkipped(t *testing.T) {
	f, rs := newFlattener(nil)
	// a .versions dir create carries no pointer yet — the update that sets it
	// follows in the same write
	dir := &filer_pb.Entry{Name: "k.versions", IsDirectory: true}
	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Empty(t, rs.deleted)
	require.Empty(t, rs.created)
}

func TestFlattenNullVersionIsLatest(t *testing.T) {
	nullObject := &filer_pb.Entry{
		Name:       "k",
		Attributes: &filer_pb.FuseAttributes{Mtime: 9},
		Chunks:     []*filer_pb.FileChunk{{FileId: "1,09"}},
	}
	f, rs := newFlattener(map[string]*filer_pb.Entry{"/buckets/b/k": nullObject})
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Same(t, nullObject, rs.created["/b/k"])
}

func TestFlattenNullVersionDeletedRescansNewestVersion(t *testing.T) {
	// deleting ?versionId=null removes the regular object without touching the
	// null-is-latest pointer: the flattened key must fall back to the newest
	// remaining version, the same rescan the S3 read path does
	older := &filer_pb.Entry{Name: "v_00000000000000010000000000000000", Chunks: []*filer_pb.FileChunk{{FileId: "1,01"}}}
	newer := &filer_pb.Entry{Name: "v_00000000000000020000000000000000", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
	f, rs := newFlattener(map[string]*filer_pb.Entry{
		"/buckets/b/k.versions/" + older.Name: older,
		"/buckets/b/k.versions/" + newer.Name: newer,
	})
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Same(t, newer, rs.created["/b/k"])
	require.Empty(t, rs.deleted)
}

func TestFlattenNullVersionDeletedNewestIsDeleteMarker(t *testing.T) {
	marker := &filer_pb.Entry{
		Name:     "v_00000000000000020000000000000000",
		Extended: map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte("true")},
	}
	older := &filer_pb.Entry{Name: "v_00000000000000010000000000000000", Chunks: []*filer_pb.FileChunk{{FileId: "1,01"}}}
	f, rs := newFlattener(map[string]*filer_pb.Entry{
		"/buckets/b/k.versions/" + older.Name:  older,
		"/buckets/b/k.versions/" + marker.Name: marker,
	})
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Equal(t, []string{"/b/k"}, rs.deleted)
	require.Empty(t, rs.created)
}

func TestFlattenNullVersionDeletedNoVersionsLeftDeletes(t *testing.T) {
	f, rs := newFlattener(map[string]*filer_pb.Entry{})
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Equal(t, []string{"/b/k"}, rs.deleted)
}

func TestFlattenNullVersionDeletedScansPastOnePage(t *testing.T) {
	// old-format version names sort oldest-first and the filer caps one
	// listing at DirListingLimit, so with a long history the newest version
	// sits beyond the first page — the rescan must read every page
	entries := map[string]*filer_pb.Entry{}
	var newest *filer_pb.Entry
	for i := 1; i <= filer.PaginationSize+1; i++ {
		v := &filer_pb.Entry{
			Name:   fmt.Sprintf("v_%032x", i),
			Chunks: []*filer_pb.FileChunk{{FileId: fmt.Sprintf("1,%02x", i)}},
		}
		entries["/buckets/b/k.versions/"+v.Name] = v
		newest = v
	}
	f, rs := newFlattener(entries)
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtNullVersionIsLatestKey: []byte("true")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Same(t, newest, rs.created["/b/k"])
}

func TestFlattenVersionFileEventRefreshesCurrentVersion(t *testing.T) {
	// a tag-only update touches the version file, not the pointer: re-resolving
	// the pointer must still upload the current version to the flat key
	version := &filer_pb.Entry{Name: "v_2", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
	}
	f, rs := newFlattener(map[string]*filer_pb.Entry{
		"/buckets/b/k.versions":     dir,
		"/buckets/b/k.versions/v_2": version,
	})

	tagUpdate := &filer_pb.Entry{Name: "v_2", Extended: map[string][]byte{"X-Amz-Tagging-env": []byte("prod")}}
	handled, err := f.process(event("/buckets/b/k.versions", "/buckets/b/k.versions", tagUpdate, tagUpdate))
	require.NoError(t, err)
	require.True(t, handled)
	require.Same(t, version, rs.created["/b/k"])
}

func TestFlattenExcludedObjectNameIsNotCopiedOrDeleted(t *testing.T) {
	f, rs := newFlattener(nil)
	f.excludeFileNames = wildcard.CompileWildcardMatchers("*.tmp")

	dir := &filer_pb.Entry{
		Name:        "notes.tmp.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionIsDeleteMarker: []byte("true")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", dir, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Empty(t, rs.deleted, "delete on the excluded flat key must be skipped")

	handled, err = f.process(event("/buckets/b", "", dir, nil))
	require.NoError(t, err)
	require.True(t, handled)
	require.Empty(t, rs.deleted)
}

func TestFlattenRegularPathFollowsVersionsPointer(t *testing.T) {
	// deleting the null object must not remove the flattened key while the
	// .versions pointer names a live version
	version := &filer_pb.Entry{Name: "v_2", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
	}
	f, rs := newFlattener(map[string]*filer_pb.Entry{
		"/buckets/b/k.versions":     dir,
		"/buckets/b/k.versions/v_2": version,
	})

	handled, err := f.process(event("/buckets/b", "", &filer_pb.Entry{Name: "k"}, nil))
	require.NoError(t, err)
	require.True(t, handled)
	require.Same(t, version, rs.created["/b/k"])
	require.Empty(t, rs.deleted)
}

func TestFlattenRenameLeavesDeleteForGenericPath(t *testing.T) {
	// rename a -> b where only b carries a .versions directory: the flattener
	// refreshes b through the pointer but the old flattened key held the moved
	// file, so the event must reach the generic path as a lone delete of a
	version := &filer_pb.Entry{Name: "v_9", Chunks: []*filer_pb.FileChunk{{FileId: "1,09"}}}
	dir := &filer_pb.Entry{
		Name:        "b.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_9")},
	}
	f, rs := newFlattener(map[string]*filer_pb.Entry{
		"/buckets/b/b.versions":     dir,
		"/buckets/b/b.versions/v_9": version,
	})

	resp := event("/buckets/b", "/buckets/b", &filer_pb.Entry{Name: "a"}, &filer_pb.Entry{Name: "b"})
	handled, err := f.process(resp)
	require.NoError(t, err)
	require.False(t, handled, "old side has no .versions sibling — generic path must delete a")
	require.NotNil(t, resp.EventNotification.OldEntry)
	require.Nil(t, resp.EventNotification.NewEntry)
	require.Same(t, version, rs.created["/b/b"])
}

func TestFlattenRenameLeavesCreateForGenericPath(t *testing.T) {
	// rename a -> b where only a carries a .versions directory: the pointer
	// still decides flattened a, so the generic path must see a lone create
	// of b and never delete the flattened old key
	version := &filer_pb.Entry{Name: "v_2", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
	dir := &filer_pb.Entry{
		Name:        "a.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
	}
	f, rs := newFlattener(map[string]*filer_pb.Entry{
		"/buckets/b/a.versions":     dir,
		"/buckets/b/a.versions/v_2": version,
	})

	resp := event("/buckets/b", "/buckets/b", &filer_pb.Entry{Name: "a"}, &filer_pb.Entry{Name: "b"})
	handled, err := f.process(resp)
	require.NoError(t, err)
	require.False(t, handled)
	require.Nil(t, resp.EventNotification.OldEntry)
	require.NotNil(t, resp.EventNotification.NewEntry)
	require.Same(t, version, rs.created["/b/a"])
	require.Empty(t, rs.deleted)
}

func TestFlattenRenameBothSidesVersioned(t *testing.T) {
	versionA := &filer_pb.Entry{Name: "v_2", Chunks: []*filer_pb.FileChunk{{FileId: "1,02"}}}
	versionB := &filer_pb.Entry{Name: "v_9", Chunks: []*filer_pb.FileChunk{{FileId: "1,09"}}}
	f, rs := newFlattener(map[string]*filer_pb.Entry{
		"/buckets/b/a.versions": {
			Name:        "a.versions",
			IsDirectory: true,
			Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_2")},
		},
		"/buckets/b/a.versions/v_2": versionA,
		"/buckets/b/b.versions": {
			Name:        "b.versions",
			IsDirectory: true,
			Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_9")},
		},
		"/buckets/b/b.versions/v_9": versionB,
	})

	resp := event("/buckets/b", "/buckets/b", &filer_pb.Entry{Name: "a"}, &filer_pb.Entry{Name: "b"})
	handled, err := f.process(resp)
	require.NoError(t, err)
	require.True(t, handled)
	require.NotNil(t, resp.EventNotification.OldEntry)
	require.NotNil(t, resp.EventNotification.NewEntry)
	require.Same(t, versionA, rs.created["/b/a"])
	require.Same(t, versionB, rs.created["/b/b"])
}

func TestFlattenDeleteMarkerVersionEntry(t *testing.T) {
	marker := &filer_pb.Entry{
		Name:     "v_3",
		Extended: map[string][]byte{s3_constants.ExtDeleteMarkerKey: []byte("true")},
	}
	f, rs := newFlattener(map[string]*filer_pb.Entry{"/buckets/b/k.versions/v_3": marker})
	dir := &filer_pb.Entry{
		Name:        "k.versions",
		IsDirectory: true,
		Extended:    map[string][]byte{s3_constants.ExtLatestVersionFileNameKey: []byte("v_3")},
	}
	handled, err := f.process(event("/buckets/b", "/buckets/b", nil, dir))
	require.NoError(t, err)
	require.True(t, handled)
	require.Equal(t, []string{"/b/k"}, rs.deleted)
}