SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package replication
import (
"context"
"testing"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var _ sink.ReplicationSink = (*recordingSink)(nil)
type deleteCall struct {
key string
isDirectory bool
}
type createCall struct {
key string
}
type updateCall struct {
key string
newParentPath string
}
type recordingSink struct {
name string
sinkToDirectory string
incremental bool
updateFoundExisting bool
deleteCalls []deleteCall
createCalls []createCall
updateCalls []updateCall
// ordered records the sink method names in call order so tests can assert
// create-before-delete sequencing.
ordered []string
}
func (s *recordingSink) GetName() string {
return s.name
}
func (s *recordingSink) Initialize(util.Configuration, string) error {
return nil
}
func (s *recordingSink) DeleteEntry(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
s.deleteCalls = append(s.deleteCalls, deleteCall{key: key, isDirectory: isDirectory})
s.ordered = append(s.ordered, "delete")
return nil
}
func (s *recordingSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []int32) error {
s.createCalls = append(s.createCalls, createCall{key: key})
s.ordered = append(s.ordered, "create")
return nil
}
func (s *recordingSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParentPath string, newEntry *filer_pb.Entry, deleteIncludeChunks bool, signatures []int32) (bool, error) {
s.updateCalls = append(s.updateCalls, updateCall{key: key, newParentPath: newParentPath})
s.ordered = append(s.ordered, "update")
return s.updateFoundExisting, nil
}
func (s *recordingSink) GetSinkToDirectory() string {
return s.sinkToDirectory
}
func (s *recordingSink) GetDestinationIdentity() string {
return s.sinkToDirectory
}
func (s *recordingSink) SetSourceFiler(*source.FilerSource) {}
func (s *recordingSink) IsIncremental() bool {
return s.incremental
}
type moveCall struct {
oldKey string
newKey string
}
// movingSink is a recordingSink that also implements sink.EntryMover, modeling
// a sink (like the filer) with a native atomic move.
type movingSink struct {
*recordingSink
moveCalls []moveCall
}
var _ sink.EntryMover = (*movingSink)(nil)
func (s *movingSink) MoveEntry(oldKey, newKey string, oldEntry, newEntry *filer_pb.Entry, signatures []int32) error {
s.moveCalls = append(s.moveCalls, moveCall{oldKey: oldKey, newKey: newKey})
s.ordered = append(s.ordered, "move")
return nil
}
func TestReplicateRenameUsesTargetKeyForNonFilerSink(t *testing.T) {
s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/source"},
}
err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "renamed.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/source/new",
})
if err != nil {
t.Fatalf("Replicate rename: %v", err)
}
if len(s.updateCalls) != 0 {
t.Fatalf("expected rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
}
if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
}
if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
}
if got, want := s.ordered, []string{"create", "delete"}; !equalStrings(got, want) {
t.Fatalf("call order = %v, want create before delete %v", got, want)
}
}
// A real move to a filer sink also goes through create-then-delete (the
// filer-sink special-case that previously routed renames to UpdateEntry is
// gone): UpdateEntry cannot move an entry across paths.
// A sink without a native move falls back to create-then-delete for a rename.
func TestReplicateRenameWithoutMoverUsesCreateThenDelete(t *testing.T) {
s := &recordingSink{
name: "s3",
sinkToDirectory: "/dest",
updateFoundExisting: true,
}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/source"},
}
err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "renamed.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/source/new",
})
if err != nil {
t.Fatalf("Replicate rename: %v", err)
}
if len(s.updateCalls) != 0 {
t.Fatalf("expected rename to bypass UpdateEntry, got %d calls", len(s.updateCalls))
}
if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/new/renamed.txt" {
t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
}
if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
}
if got, want := s.ordered, []string{"create", "delete"}; !equalStrings(got, want) {
t.Fatalf("call order = %v, want create before delete %v", got, want)
}
}
// A sink with a native move (the filer) relocates a rename via MoveEntry — never
// create-then-delete, so a failed copy can't delete the only valid destination.
func TestReplicateRenameUsesMoveEntryWhenSupported(t *testing.T) {
s := &movingSink{recordingSink: &recordingSink{name: "filer", sinkToDirectory: "/dest"}}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/source"},
}
err := r.Replicate(context.Background(), "/source/old/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{Name: "file.txt"},
NewEntry: &filer_pb.Entry{Name: "renamed.txt"},
NewParentPath: "/source/new",
})
if err != nil {
t.Fatalf("Replicate rename: %v", err)
}
if len(s.moveCalls) != 1 || s.moveCalls[0].oldKey != "/dest/old/file.txt" || s.moveCalls[0].newKey != "/dest/new/renamed.txt" {
t.Fatalf("move calls = %+v, want one move /dest/old/file.txt => /dest/new/renamed.txt", s.moveCalls)
}
if len(s.createCalls) != 0 || len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
t.Fatalf("native move must not create/delete/update: creates=%v deletes=%v updates=%v", s.createCalls, s.deleteCalls, s.updateCalls)
}
}
// An in-place update (same parent + same name, so oldSinkKey == newSinkKey)
// still routes to UpdateEntry rather than create-then-delete.
func TestReplicateInPlaceUpdateUsesUpdateEntry(t *testing.T) {
s := &recordingSink{
name: "filer",
sinkToDirectory: "/dest",
updateFoundExisting: true,
}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/source"},
}
err := r.Replicate(context.Background(), "/source/dir/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 456,
},
},
NewParentPath: "/source/dir",
})
if err != nil {
t.Fatalf("Replicate in-place update: %v", err)
}
if len(s.updateCalls) != 1 {
t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
}
if s.updateCalls[0].key != "/dest/dir/file.txt" {
t.Fatalf("update key = %q, want /dest/dir/file.txt", s.updateCalls[0].key)
}
if s.updateCalls[0].newParentPath != "/dest/dir" {
t.Fatalf("update newParentPath = %q, want /dest/dir", s.updateCalls[0].newParentPath)
}
if len(s.deleteCalls) != 0 || len(s.createCalls) != 0 {
t.Fatalf("unexpected delete/create calls: deletes=%+v creates=%+v", s.deleteCalls, s.createCalls)
}
}
// When the in-place update finds no existing entry to update, fall back to
// delete-then-create at the same key.
func TestReplicateInPlaceUpdateFallbackCreates(t *testing.T) {
s := &recordingSink{
name: "filer",
sinkToDirectory: "/dest",
updateFoundExisting: false,
}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/source"},
}
err := r.Replicate(context.Background(), "/source/dir/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 456,
},
},
NewParentPath: "/source/dir",
})
if err != nil {
t.Fatalf("Replicate in-place update fallback: %v", err)
}
if len(s.updateCalls) != 1 {
t.Fatalf("update calls = %d, want 1", len(s.updateCalls))
}
if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/dir/file.txt" {
t.Fatalf("delete calls = %+v, want same sink key", s.deleteCalls)
}
if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/dir/file.txt" {
t.Fatalf("create calls = %+v, want same sink key", s.createCalls)
}
if got, want := s.ordered, []string{"update", "delete", "create"}; !equalStrings(got, want) {
t.Fatalf("call order = %v, want %v", got, want)
}
}
func equalStrings(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func TestPathIsEqualOrUnderUsesDirectoryBoundaries(t *testing.T) {
tests := []struct {
name string
candidate string
other string
expected bool
}{
{name: "equal", candidate: "/foo", other: "/foo", expected: true},
{name: "descendant", candidate: "/foo/bar", other: "/foo", expected: true},
{name: "sibling prefix", candidate: "/foobar/bar", other: "/foo", expected: false},
{name: "root", candidate: "/foo/bar", other: "/", expected: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := util.IsEqualOrUnder(tt.candidate, tt.other); got != tt.expected {
t.Fatalf("IsEqualOrUnder(%q, %q) = %v, want %v", tt.candidate, tt.other, got, tt.expected)
}
})
}
}
func TestReplicateRenameOutToSiblingPrefixBecomesDelete(t *testing.T) {
s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/foo"},
}
err := r.Replicate(context.Background(), "/foo/old/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/foobar/new",
})
if err != nil {
t.Fatalf("Replicate rename out to sibling prefix: %v", err)
}
if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/old/file.txt" {
t.Fatalf("delete calls = %+v, want old sink key", s.deleteCalls)
}
if len(s.createCalls) != 0 || len(s.updateCalls) != 0 {
t.Fatalf("unexpected create/update calls: creates=%+v updates=%+v", s.createCalls, s.updateCalls)
}
}
func TestReplicateRenameFromExcludedDirBecomesCreate(t *testing.T) {
s := &recordingSink{name: "local", sinkToDirectory: "/dest"}
r := &Replicator{
sink: s,
source: &source.FilerSource{Dir: "/foo"},
excludeDirs: []string{"/foo/excluded"},
}
err := r.Replicate(context.Background(), "/foo/excluded/file.txt", &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewEntry: &filer_pb.Entry{
Name: "file.txt",
Attributes: &filer_pb.FuseAttributes{
Mtime: 123,
},
},
NewParentPath: "/foo/live",
})
if err != nil {
t.Fatalf("Replicate rename from excluded dir: %v", err)
}
if len(s.createCalls) != 1 || s.createCalls[0].key != "/dest/live/file.txt" {
t.Fatalf("create calls = %+v, want target sink key", s.createCalls)
}
if len(s.deleteCalls) != 0 || len(s.updateCalls) != 0 {
t.Fatalf("unexpected delete/update calls: deletes=%+v updates=%+v", s.deleteCalls, s.updateCalls)
}
}
type keepingSink struct {
*recordingSink
keepCalls []deleteCall
}
var _ sink.MetadataOnlyDeleter = (*keepingSink)(nil)
func (s *keepingSink) DeleteEntryKeepingRemoteObject(key string, isDirectory, deleteIncludeChunks bool, signatures []int32) error {
s.keepCalls = append(s.keepCalls, deleteCall{key: key, isDirectory: isDirectory})
return nil
}
func TestReplicateMetadataOnlyDeleteKeepsTheRemoteObject(t *testing.T) {
s := &keepingSink{recordingSink: &recordingSink{name: "filer", sinkToDirectory: "/dest"}}
r := &Replicator{sink: s, source: &source.FilerSource{Dir: "/source"}}
deleteOf := func(name string, extended map[string][]byte) *filer_pb.EventNotification {
return &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}, Extended: extended}}
}
if err := r.Replicate(context.Background(), "/source/dir/kept.bin", deleteOf("kept.bin", map[string][]byte{filer.ExtKeepRemoteObjectKey: []byte("true")})); err != nil {
t.Fatal(err)
}
if err := r.Replicate(context.Background(), "/source/dir/gone.bin", deleteOf("gone.bin", nil)); err != nil {
t.Fatal(err)
}
if len(s.keepCalls) != 1 || s.keepCalls[0].key != "/dest/dir/kept.bin" {
t.Fatalf("metadata-only deletes = %+v, want /dest/dir/kept.bin", s.keepCalls)
}
if len(s.deleteCalls) != 1 || s.deleteCalls[0].key != "/dest/dir/gone.bin" {
t.Fatalf("ordinary deletes = %+v, want /dest/dir/gone.bin", s.deleteCalls)
}
}
type versionedSink struct {
*recordingSink
versionCalls []*filer_pb.Entry
}
var _ sink.VersionedDeleter = (*versionedSink)(nil)
func (s *versionedSink) DeleteEntryVersion(key string, oldEntry *filer_pb.Entry, tsNs int64, deleteIncludeChunks bool, signatures []int32) error {
s.versionCalls = append(s.versionCalls, oldEntry)
return s.DeleteEntry(key, oldEntry.IsDirectory, deleteIncludeChunks, signatures)
}
func TestReplicateDeletesHandTheDeletedVersionToTheSink(t *testing.T) {
s := &versionedSink{recordingSink: &recordingSink{name: "local", sinkToDirectory: "/dest"}}
r := &Replicator{sink: s, source: &source.FilerSource{Dir: "/source"}}
deleted := &filer_pb.Entry{Name: "gone.bin", CommitTsNs: 100, Attributes: &filer_pb.FuseAttributes{}}
if err := r.Replicate(context.Background(), "/source/dir/gone.bin", &filer_pb.EventNotification{OldEntry: deleted}); err != nil {
t.Fatal(err)
}
renamed := &filer_pb.Entry{Name: "old.bin", CommitTsNs: 200, Attributes: &filer_pb.FuseAttributes{}}
if err := r.Replicate(context.Background(), "/source/dir/old.bin", &filer_pb.EventNotification{
OldEntry: renamed,
NewEntry: &filer_pb.Entry{Name: "new.bin", Attributes: &filer_pb.FuseAttributes{}},
NewParentPath: "/source/dir",
}); err != nil {
t.Fatal(err)
}
if len(s.versionCalls) != 2 || s.versionCalls[0] != deleted || s.versionCalls[1] != renamed {
t.Fatalf("versioned deletes = %v, want the deleted and the renamed entries", s.versionCalls)
}
if len(s.deleteCalls) != 2 || s.deleteCalls[0].key != "/dest/dir/gone.bin" || s.deleteCalls[1].key != "/dest/dir/old.bin" {
t.Fatalf("deletes = %+v", s.deleteCalls)
}
}