Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

10.1 KB
Raw Blame
package s3api

import (
	"context"
	"fmt"
	"math"
	"path"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
	"github.com/seaweedfs/seaweedfs/weed/wdclient"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"
)

func ambiguousRouteServer(t *testing.T, addrs ...pb.ServerAddress) *S3ApiServer {
	t.Helper()
	dialOption := grpc.WithTransportCredentials(insecure.NewCredentials())
	return &S3ApiServer{
		option:      &S3ApiServerOption{GrpcDialOption: dialOption, Filers: addrs},
		filerClient: wdclient.NewFilerClient(addrs, dialOption, ""),
	}
}

func TestCreateAfterAmbiguousRoute(t *testing.T) {
	uploaded := []*filer_pb.FileChunk{{FileId: "5,01637037d6", Size: 55}}
	filePath := "/buckets/b/o"
	chunksAlive := func(context.Context, []*filer_pb.FileChunk) bool { return true }
	chunksDead := func(context.Context, []*filer_pb.FileChunk) bool { return false }
	newS3a := func(t *testing.T, f *fakeLookupFiler) (*S3ApiServer, pb.ServerAddress) {
		addr := startFakeFiler(t, f)
		return ambiguousRouteServer(t, addr), addr
	}

	t.Run("stored entry is this PUT's — route landed", func(t *testing.T) {
		f := &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", Chunks: []*filer_pb.FileChunk{{FileId: "5,01637037d6", Size: 55}}}}
		s3a, owner := newS3a(t, f)
		entryCreated := false
		ran := false
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
		if code != s3err.ErrNone || !entryCreated {
			t.Fatalf("code=%v entryCreated=%v", code, entryCreated)
		}
		if ran {
			t.Fatal("re-committed an entry the route already stored")
		}
	})

	t.Run("stored entry is a newer write — refuse stale re-commit", func(t *testing.T) {
		f := &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", Chunks: []*filer_pb.FileChunk{{FileId: "5,01637037d7", Size: 55}}}}
		s3a, owner := newS3a(t, f)
		entryCreated := false
		ran := false
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
		if code != s3err.ErrServiceUnavailable {
			t.Fatalf("code = %v, want ServiceUnavailable", code)
		}
		if ran || entryCreated {
			t.Fatal("stale entry was committed over a newer write")
		}
	})

	t.Run("stored entry is a directory, chunks alive — lock path answers", func(t *testing.T) {
		dir := &filer_pb.Entry{Name: "o", IsDirectory: true}
		f := &fakeLookupFiler{entry: dir}
		s3a, owner := newS3a(t, f)
		entryCreated := false
		ran := false
		var gotCond *filer_pb.WriteCondition
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(cond *filer_pb.WriteCondition) s3err.ErrorCode {
			ran = true
			gotCond = cond
			return s3err.ErrExistingObjectIsDirectory
		})
		if code != s3err.ErrExistingObjectIsDirectory || !ran {
			t.Fatalf("code=%v ran=%v — a directory conflict with live chunks must reach createUnderLock, which maps it", code, ran)
		}
		c := oneClause(t, gotCond)
		if c.Kind != filer_pb.WriteCondition_IF_ENTRY_EQUAL || !c.ExpectedEntry.IsDirectory {
			t.Fatalf("directory retry carried %+v, want an IF_ENTRY_EQUAL pin on the directory", c)
		}
	})

	// The routed PUT may have committed before a delete removed the entry and
	// its chunks and a nested write recreated the name as a directory; dead
	// chunks mean re-committing would store an entry pointing at them.
	t.Run("stored entry is a directory, chunks unverifiable — refuse", func(t *testing.T) {
		f := &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", IsDirectory: true}}
		s3a, owner := newS3a(t, f)
		entryCreated := false
		ran := false
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksDead, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
		if code != s3err.ErrServiceUnavailable || ran {
			t.Fatalf("code=%v ran=%v — re-committed over a directory with possibly-deleted chunks", code, ran)
		}
	})

	t.Run("match behind an unanswered filer — refuse", func(t *testing.T) {
		deadAddr := startFakeFiler(t, &fakeLookupFiler{lookupErr: context.DeadlineExceeded})
		matchAddr := startFakeFiler(t, &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", Chunks: []*filer_pb.FileChunk{{FileId: "5,01637037d6", Size: 55}}}})
		s3a := ambiguousRouteServer(t, deadAddr, matchAddr)
		entryCreated := false
		ran := false
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", deadAddr, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
		if code != s3err.ErrServiceUnavailable || entryCreated || ran {
			t.Fatalf("code=%v entryCreated=%v ran=%v — a match behind an unresolved lookup must not finalize", code, entryCreated, ran)
		}
	})

	t.Run("entry proven absent, chunks alive — conditional create proceeds", func(t *testing.T) {
		f := &fakeLookupFiler{}
		s3a, owner := newS3a(t, f)
		entryCreated := false
		ran := false
		var gotCond *filer_pb.WriteCondition
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(cond *filer_pb.WriteCondition) s3err.ErrorCode { ran = true; gotCond = cond; return s3err.ErrNone })
		if code != s3err.ErrNone || !ran {
			t.Fatalf("code=%v ran=%v — proven-absent entry did not reach createUnderLock", code, ran)
		}
		if c := oneClause(t, gotCond); c.Kind != filer_pb.WriteCondition_IF_NOT_EXISTS {
			t.Fatalf("absent-entry create carried %+v, want IF_NOT_EXISTS", c)
		}
	})

	t.Run("entry absent but chunks unverifiable — refuse", func(t *testing.T) {
		f := &fakeLookupFiler{}
		s3a, owner := newS3a(t, f)
		entryCreated := false
		ran := false
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksDead, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
		if code != s3err.ErrServiceUnavailable || ran {
			t.Fatalf("code=%v ran=%v — re-committed an entry over possibly-deleted chunks", code, ran)
		}
	})

	t.Run("lookup cannot resolve state — refuse", func(t *testing.T) {
		f := &fakeLookupFiler{lookupErr: context.DeadlineExceeded}
		s3a, owner := newS3a(t, f)
		entryCreated := false
		ran := false
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, nil, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
		if code != s3err.ErrServiceUnavailable || ran {
			t.Fatalf("code=%v ran=%v — uncertain state still committed", code, ran)
		}
	})

	t.Run("finalize fails — recovered entry rolls back", func(t *testing.T) {
		f := &fakeLookupFiler{entry: &filer_pb.Entry{Name: "o", Chunks: []*filer_pb.FileChunk{{FileId: "5,01637037d6", Size: 55}}}}
		s3a, owner := newS3a(t, f)
		entryCreated := false
		ran := false
		finalize := &putFinalize{afterCreate: func(*filer_pb.Entry) s3err.ErrorCode { return s3err.ErrInternalError }}
		code := s3a.createAfterAmbiguousRoute(filePath, "b", "o", owner, &filer_pb.Entry{Name: "o"}, uploaded, finalize, &entryCreated, chunksAlive, func(*filer_pb.WriteCondition) s3err.ErrorCode { ran = true; return s3err.ErrNone })
		if code != s3err.ErrInternalError || ran {
			t.Fatalf("code=%v ran=%v", code, ran)
		}
		if len(f.deleted) != 1 || f.deleted[0] != filePath {
			t.Fatalf("recovered entry was not rolled back: deleted=%v", f.deleted)
		}
	})
}

type fakeVersionedDirFiler struct {
	filer_pb.UnimplementedSeaweedFilerServer
	entries map[string]*filer_pb.Entry
	updates map[string]*filer_pb.Entry
}

func (f *fakeVersionedDirFiler) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
	if e, ok := f.entries[path.Join(req.Directory, req.Name)]; ok {
		return &filer_pb.LookupDirectoryEntryResponse{Entry: e}, nil
	}
	return nil, filer_pb.ErrNotFound
}

func (f *fakeVersionedDirFiler) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
	f.updates[path.Join(req.Directory, req.Entry.Name)] = req.Entry
	return &filer_pb.UpdateEntryResponse{}, nil
}

// A late versioned write keeps the newer latest pointer, but the version it
// just stored is born noncurrent — without a NoncurrentSinceNs stamp the
// lifecycle engine never ages it out.
func TestUpdateLatestVersionInDirectoryBornNoncurrent(t *testing.T) {
	now := time.Now().UnixNano()
	newerId := fmt.Sprintf("%016x%s", math.MaxInt64-now, "1111111111111111")
	olderId := fmt.Sprintf("%016x%s", math.MaxInt64-(now-int64(time.Hour)), "2222222222222222")

	bucketDir := "/buckets/b"
	versionsPath := path.Join(bucketDir, "o"+s3_constants.VersionsFolder)
	f := &fakeVersionedDirFiler{
		entries: map[string]*filer_pb.Entry{
			versionsPath: {
				Name:        "o" + s3_constants.VersionsFolder,
				IsDirectory: true,
				Extended: map[string][]byte{
					s3_constants.ExtLatestVersionIdKey:       []byte(newerId),
					s3_constants.ExtLatestVersionFileNameKey: []byte("newer.v"),
				},
			},
			path.Join(versionsPath, "late.v"): {Name: "late.v"},
		},
		updates: map[string]*filer_pb.Entry{},
	}
	s3a := ambiguousRouteServer(t, startFakeFiler(t, f))
	s3a.option.BucketsPath = "/buckets"

	err := s3a.updateLatestVersionInDirectory("b", "o", olderId, "late.v", &filer_pb.Entry{})
	if err != nil {
		t.Fatalf("updateLatestVersionInDirectory: %v", err)
	}
	stamped := f.updates[path.Join(versionsPath, "late.v")]
	if stamped == nil {
		t.Fatal("late version never got its noncurrent stamp")
	}
	if stamped.Extended[s3_constants.ExtNoncurrentSinceNsKey] == nil {
		t.Fatal("stamp did not set ExtNoncurrentSinceNsKey")
	}
	if _, pointerTouched := f.updates[versionsPath]; pointerTouched {
		t.Fatal("the newer latest pointer must not move")
	}
}