Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

5.0 KB
Raw Blame
package meta_cache

import (
	"context"
	"errors"
	"fmt"
	"testing"

	"github.com/seaweedfs/seaweedfs/weed/filer"

	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/grpc"
)

func listResponses(n int) []*filer_pb.ListEntriesResponse {
	responses := make([]*filer_pb.ListEntriesResponse, 0, n)
	for i := 0; i < n; i++ {
		responses = append(responses, &filer_pb.ListEntriesResponse{
			Entry: &filer_pb.Entry{
				Name: fmt.Sprintf("f%05d", i),
				Attributes: &filer_pb.FuseAttributes{
					Crtime: 1, Mtime: 1, FileMode: 0o644, FileSize: 3,
				},
			},
		})
	}
	return responses
}

// TestEnsureVisitedRefusesOversizedDirectory checks that a directory past the
// limit is not cached, that the refusal is remembered, and that the partial
// build leaves nothing behind in the local store.
func TestEnsureVisitedRefusesOversizedDirectory(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
	defer mc.Shutdown()

	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(10)}}

	err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5)
	var tooLarge *DirectoryTooLargeError
	if !errors.As(err, &tooLarge) {
		t.Fatalf("EnsureVisited = %v, want DirectoryTooLargeError", err)
	}
	if tooLarge.Path != util.FullPath("/dir") {
		t.Fatalf("oversized path = %s, want /dir", tooLarge.Path)
	}
	if mc.IsDirectoryCached(util.FullPath("/dir")) {
		t.Error("oversized directory reported cached")
	}
	// The aborted build must leave no partial children to be served later.
	count := 0
	if _, err := mc.ListDirectoryEntries(context.Background(), util.FullPath("/dir"), "", false, 100, func(e *filer.Entry) (bool, error) {
		count++
		return true, nil
	}); err != nil {
		t.Fatalf("list: %v", err)
	}
	if count != 0 {
		t.Errorf("local store still holds %d children of the aborted build", count)
	}

	// A second visit fails fast without streaming to the limit again.
	if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5); !errors.As(err, &tooLarge) {
		t.Fatalf("second EnsureVisited = %v, want DirectoryTooLargeError", err)
	}
}

// TestEnsureVisitedStepsOverOversizedAncestor checks a huge ancestor does not
// wedge the caching of its subdirectories.
func TestEnsureVisitedStepsOverOversizedAncestor(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
	defer mc.Shutdown()
	mc.markOversized(util.FullPath("/huge"))

	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(3)}}
	if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
		t.Fatalf("EnsureVisited under an oversized ancestor: %v", err)
	}
	if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
		t.Error("subdirectory of an oversized ancestor was not cached")
	}
	if mc.IsDirectoryCached(util.FullPath("/huge")) {
		t.Error("oversized ancestor became cached")
	}
}

// TestEnsureVisitedUnderTheLimitStillCaches pins that the gate does not change
// behaviour for ordinary directories.
func TestEnsureVisitedUnderTheLimitStillCaches(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
	defer mc.Shutdown()

	accessor := &buildFilerAccessor{client: &buildListClient{responses: listResponses(5)}}
	if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/dir"), 5); err != nil {
		t.Fatalf("EnsureVisited: %v", err)
	}
	if !mc.IsDirectoryCached(util.FullPath("/dir")) {
		t.Error("directory at the limit was not cached")
	}
}

// pathListClient serves canned listings per directory, so one visit can see
// directories of different sizes.
type pathListClient struct {
	filer_pb.SeaweedFilerClient
	perDir map[string][]*filer_pb.ListEntriesResponse
}

func (c *pathListClient) ListEntries(ctx context.Context, in *filer_pb.ListEntriesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[filer_pb.ListEntriesResponse], error) {
	return &buildListStream{responses: c.perDir[in.Directory]}, nil
}

// TestEnsureVisitedAncestorFoundOversizedMidVisit covers the first discovery:
// the ancestor's refusal must neither cancel the descendant's build nor be
// reported as the descendant's own.
func TestEnsureVisitedAncestorFoundOversizedMidVisit(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true})
	defer mc.Shutdown()

	accessor := &buildFilerAccessor{client: &pathListClient{perDir: map[string][]*filer_pb.ListEntriesResponse{
		"/huge":     listResponses(10),
		"/huge/sub": listResponses(3),
	}}}

	if err := EnsureVisited(context.Background(), mc, accessor, util.FullPath("/huge/sub"), 5); err != nil {
		t.Fatalf("EnsureVisited: %v", err)
	}
	if !mc.IsDirectoryCached(util.FullPath("/huge/sub")) {
		t.Error("descendant of a just-discovered oversized ancestor was not cached")
	}
	if mc.IsDirectoryCached(util.FullPath("/huge")) {
		t.Error("oversized ancestor became cached")
	}
	if !mc.isOversized(util.FullPath("/huge")) {
		t.Error("oversized ancestor was not remembered")
	}
}