Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

15.1 KB
Raw Blame
package replication

import (
	"context"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/replication/sink"
	"github.com/seaweedfs/seaweedfs/weed/replication/source"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

var _ sink.ReplicationSink = (*recordingSink)(nil)

type deleteCall struct {
	key         string
	isDirectory bool
}

type createCall struct {
	key string
}

type updateCall struct {
	key           string
	newParentPath string
}

type recordingSink struct {
	name                string
	sinkToDirectory     string
	incremental         bool
	updateFoundExisting bool

	deleteCalls []deleteCall
	createCalls []createCall
	updateCalls []updateCall

	// ordered records the sink method names in call order so tests can assert
	// create-before-delete sequencing.
	ordered []string
}

func (s *recordingSink) GetName() string {
	return s.name
}

func (s *recordingSink) Initialize(util.Configuration, string) error {
	return nil
}

func (s *recordingSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
	s.deleteCalls = append(s.deleteCalls, deleteCall{key: key, isDirectory: isDirectory})
	s.ordered = append(s.ordered, "delete")
	return nil
}

func (s *recordingSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
	s.createCalls = append(s.createCalls, createCall{key: key})
	s.ordered = append(s.ordered, "create")
	return nil
}

func (s *recordingSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
	s.updateCalls = append(s.updateCalls, updateCall{key: key, newParentPath: newParentPath})
	s.ordered = append(s.ordered, "update")
	return s.updateFoundExisting, nil
}

func (s *recordingSink) GetSinkToDirectory() string {
	return s.sinkToDirectory
}

func (s *recordingSink) GetDestinationIdentity() string {
	return s.sinkToDirectory
}

func (s *recordingSink) SetSourceFiler(*source.FilerSource) {}

func (s *recordingSink) IsIncremental() bool {
	return s.incremental
}

type moveCall struct {
	oldKey string
	newKey string
}

// movingSink is a recordingSink that also implements sink.EntryMover, modeling
// a sink (like the filer) with a native atomic move.
type movingSink struct {
	*recordingSink
	moveCalls []moveCall
}

var _ sink.EntryMover = (*movingSink)(nil)

func (s *movingSink) MoveEntry(oldKey, newKey string, oldEntry, newEntry *filer_pb.Entry, signatures []int32) error {
	s.moveCalls = append(s.moveCalls, moveCall{oldKey: oldKey, newKey: newKey})
	s.ordered = append(s.ordered, "move")
	return nil
}

func TestReplicateRenameUsesTargetKeyForNonFilerSink(t *testing.T) {
	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "renamed.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewParentPath: "/source/new",
	})
	if err != nil {
		t.Fatalf("Replicate rename: %v", err)
	}

	if len(s.updateCalls) != 0 {
		t.Fatalf("expected rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
	}
	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
	}
	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
	}
	if got, want := s.ordered, []string{"create", "delete"}; !equalStrings(got, want) {
		t.Fatalf("call order = %v, want create before delete %v", got, want)
	}
}

// A real move to a filer sink also goes through create-then-delete (the
// filer-sink special-case that previously routed renames to UpdateEntry is
// gone): UpdateEntry cannot move an entry across paths.
// A sink without a native move falls back to create-then-delete for a rename.
func TestReplicateRenameWithoutMoverUsesCreateThenDelete(t *testing.T) {
	s := &recordingSink{
		name:                "s3",
		sinkToDirectory:     "/dest",
		updateFoundExisting: true,
	}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "renamed.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewParentPath: "/source/new",
	})
	if err != nil {
		t.Fatalf("Replicate rename: %v", err)
	}

	if len(s.updateCalls) != 0 {
		t.Fatalf("expected rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
	}
	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
	}
	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
	}
	if got, want := s.ordered, []string{"create", "delete"}; !equalStrings(got, want) {
		t.Fatalf("call order = %v, want create before delete %v", got, want)
	}
}

// A sink with a native move (the filer) relocates a rename via MoveEntry — never
// create-then-delete, so a failed copy can't delete the only valid destination.
func TestReplicateRenameUsesMoveEntryWhenSupported(t *testing.T) {
	s := &movingSink{recordingSink: &recordingSink{name: "filer", sinkToDirectory: "/dest"}}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
		OldEntry:      &filer_pb.Entry{Name: "file.txt"},
		NewEntry:      &filer_pb.Entry{Name: "renamed.txt"},
		NewParentPath: "/source/new",
	})
	if err != nil {
		t.Fatalf("Replicate rename: %v", err)
	}

	if len(s.moveCalls) != 1 || s.moveCalls[0].oldKey != "/dest/old/file.txt" || s.moveCalls[0].newKey != "/dest/new/renamed.txt" {
		t.Fatalf("move calls = %+v, want one move /dest/old/file.txt => /dest/new/renamed.txt", s.moveCalls)
	}
	if len(s.createCalls) != 0 || len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
		t.Fatalf("native move must not create/delete/update: creates=%v deletes=%v updates=%v", s.createCalls, s.deleteCalls, s.updateCalls)
	}
}

// An in-place update (same parent + same name, so oldSinkKey == newSinkKey)
// still routes to UpdateEntry rather than create-then-delete.
func TestReplicateInPlaceUpdateUsesUpdateEntry(t *testing.T) {
	s := &recordingSink{
		name:                "filer",
		sinkToDirectory:     "/dest",
		updateFoundExisting: true,
	}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/dir/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 456,
			},
		},
		NewParentPath: "/source/dir",
	})
	if err != nil {
		t.Fatalf("Replicate in-place update: %v", err)
	}

	if len(s.updateCalls) != 1 {
		t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
	}
	if s.updateCalls[0].key != "/dest/dir/file.txt" {
		t.Fatalf("update key = %q, want /dest/dir/file.txt", s.updateCalls[0].key)
	}
	if s.updateCalls[0].newParentPath != "/dest/dir" {
		t.Fatalf("update newParentPath = %q, want /dest/dir", s.updateCalls[0].newParentPath)
	}
	if len(s.deleteCalls) != 0 || len(s.createCalls) != 0 {
		t.Fatalf("unexpected delete/create calls: deletes=%+v creates=%+v", s.deleteCalls, s.createCalls)
	}
}

// When the in-place update finds no existing entry to update, fall back to
// delete-then-create at the same key.
func TestReplicateInPlaceUpdateFallbackCreates(t *testing.T) {
	s := &recordingSink{
		name:                "filer",
		sinkToDirectory:     "/dest",
		updateFoundExisting: false,
	}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/source"},
	}

	err := r.Replicate(context.Background(), "/source/dir/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 456,
			},
		},
		NewParentPath: "/source/dir",
	})
	if err != nil {
		t.Fatalf("Replicate in-place update fallback: %v", err)
	}

	if len(s.updateCalls) != 1 {
		t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
	}
	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/dir/file.txt" {
		t.Fatalf("delete calls = %+v, want same sink key", s.deleteCalls)
	}
	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/dir/file.txt" {
		t.Fatalf("create calls = %+v, want same sink key", s.createCalls)
	}
	if got, want := s.ordered, []string{"update", "delete", "create"}; !equalStrings(got, want) {
		t.Fatalf("call order = %v, want %v", got, want)
	}
}

func equalStrings(a, b []string) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

func TestPathIsEqualOrUnderUsesDirectoryBoundaries(t *testing.T) {
	tests := []struct {
		name      string
		candidate string
		other     string
		expected  bool
	}{
		{name: "equal", candidate: "/foo", other: "/foo", expected: true},
		{name: "descendant", candidate: "/foo/bar", other: "/foo", expected: true},
		{name: "sibling prefix", candidate: "/foobar/bar", other: "/foo", expected: false},
		{name: "root", candidate: "/foo/bar", other: "/", expected: true},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := util.IsEqualOrUnder(tt.candidate, tt.other); got != tt.expected {
				t.Fatalf("IsEqualOrUnder(%q, %q) = %v, want %v", tt.candidate, tt.other, got, tt.expected)
			}
		})
	}
}

func TestReplicateRenameOutToSiblingPrefixBecomesDelete(t *testing.T) {
	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
	r := &Replicator{
		sink:   s,
		source: &source.FilerSource{Dir: "/foo"},
	}

	err := r.Replicate(context.Background(), "/foo/old/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewParentPath: "/foobar/new",
	})
	if err != nil {
		t.Fatalf("Replicate rename out to sibling prefix: %v", err)
	}

	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
		t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
	}
	if len(s.createCalls) != 0 || len(s.updateCalls) != 0 {
		t.Fatalf("unexpected create/update calls: creates=%+v updates=%+v", s.createCalls, s.updateCalls)
	}
}

func TestReplicateRenameFromExcludedDirBecomesCreate(t *testing.T) {
	s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
	r := &Replicator{
		sink:        s,
		source:      &source.FilerSource{Dir: "/foo"},
		excludeDirs: []string{"/foo/excluded"},
	}

	err := r.Replicate(context.Background(), "/foo/excluded/file.txt", &filer_pb.EventNotification{
		OldEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewEntry: &filer_pb.Entry{
			Name: "file.txt",
			Attributes: &filer_pb.FuseAttributes{
				Mtime: 123,
			},
		},
		NewParentPath: "/foo/live",
	})
	if err != nil {
		t.Fatalf("Replicate rename from excluded dir: %v", err)
	}

	if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/live/file.txt" {
		t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
	}
	if len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
		t.Fatalf("unexpected delete/update calls: deletes=%+v updates=%+v", s.deleteCalls, s.updateCalls)
	}
}

type keepingSink struct {
	*recordingSink
	keepCalls []deleteCall
}

var _ sink.MetadataOnlyDeleter = (*keepingSink)(nil)

func (s *keepingSink) DeleteEntryKeepingRemoteObject(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
	s.keepCalls = append(s.keepCalls, deleteCall{key: key, isDirectory: isDirectory})
	return nil
}

func TestReplicateMetadataOnlyDeleteKeepsTheRemoteObject(t *testing.T) {
	s := &keepingSink{recordingSink: &recordingSink{name: "filer", sinkToDirectory: "/dest"}}
	r := &Replicator{sink: s, source: &source.FilerSource{Dir: "/source"}}
	deleteOf := func(name string, extended map[string][]byte) *filer_pb.EventNotification {
		return &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}, Extended: extended}}
	}

	if err := r.Replicate(context.Background(), "/source/dir/kept.bin", deleteOf("kept.bin", map[string][]byte{filer.ExtKeepRemoteObjectKey: []byte("true")})); err != nil {
		t.Fatal(err)
	}
	if err := r.Replicate(context.Background(), "/source/dir/gone.bin", deleteOf("gone.bin", nil)); err != nil {
		t.Fatal(err)
	}

	if len(s.keepCalls) != 1 || s.keepCalls[0].key != "/dest/dir/kept.bin" {
		t.Fatalf("metadata-only deletes = %+v, want /dest/dir/kept.bin", s.keepCalls)
	}
	if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/dir/gone.bin" {
		t.Fatalf("ordinary deletes = %+v, want /dest/dir/gone.bin", s.deleteCalls)
	}
}

type versionedSink struct {
	*recordingSink
	versionCalls []*filer_pb.Entry
}

var _ sink.VersionedDeleter = (*versionedSink)(nil)

func (s *versionedSink) DeleteEntryVersion(key string, oldEntry *filer_pb.Entry, tsNs int64, deleteIncludeChunks bool, signatures []int32) error {
	s.versionCalls = append(s.versionCalls, oldEntry)
	return s.DeleteEntry(key, oldEntry.IsDirectory, deleteIncludeChunks, signatures)
}

func TestReplicateDeletesHandTheDeletedVersionToTheSink(t *testing.T) {
	s := &versionedSink{recordingSink: &recordingSink{name: "local", sinkToDirectory: "/dest"}}
	r := &Replicator{sink: s, source: &source.FilerSource{Dir: "/source"}}
	deleted := &filer_pb.Entry{Name: "gone.bin", CommitTsNs: 100, Attributes: &filer_pb.FuseAttributes{}}
	if err := r.Replicate(context.Background(), "/source/dir/gone.bin", &filer_pb.EventNotification{OldEntry: deleted}); err != nil {
		t.Fatal(err)
	}
	renamed := &filer_pb.Entry{Name: "old.bin", CommitTsNs: 200, Attributes: &filer_pb.FuseAttributes{}}
	if err := r.Replicate(context.Background(), "/source/dir/old.bin", &filer_pb.EventNotification{
		OldEntry:      renamed,
		NewEntry:      &filer_pb.Entry{Name: "new.bin", Attributes: &filer_pb.FuseAttributes{}},
		NewParentPath: "/source/dir",
	}); err != nil {
		t.Fatal(err)
	}

	if len(s.versionCalls) != 2 || s.versionCalls[0] != deleted || s.versionCalls[1] != renamed {
		t.Fatalf("versioned deletes = %v, want the deleted and the renamed entries", s.versionCalls)
	}
	if len(s.deleteCalls) != 2 || s.deleteCalls[0].key != "/dest/dir/gone.bin" || s.deleteCalls[1].key != "/dest/dir/old.bin" {
		t.Fatalf("deletes = %+v", s.deleteCalls)
	}
}