Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

44.0 KB
Raw Blame
package meta_cache

import (
	"context"
	"errors"
	"fmt"
	"net"
	"sync"
	"testing"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials/insecure"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

func TestSectionOf(t *testing.T) {
	sl := newSectionTable([]string{"g", "p"})
	for name, want := range map[string]int{
		"a": 0, "f": 0,
		"g": 1, "h": 1, "o": 1,
		"p": 2, "z": 2,
	} {
		if got := sl.sectionOf(name); got != want {
			t.Errorf("sectionOf(%q) = %d, want %d", name, got, want)
		}
	}
	if lo, hi := sl.sectionRange(0); lo != "" || hi != "g" {
		t.Errorf("sectionRange(0) = %q..%q, want ..g", lo, hi)
	}
	if lo, hi := sl.sectionRange(1); lo != "g" || hi != "p" {
		t.Errorf("sectionRange(1) = %q..%q, want g..p", lo, hi)
	}
	if lo, hi := sl.sectionRange(2); lo != "p" || hi != "" {
		t.Errorf("sectionRange(2) = %q..%q, want p..", lo, hi)
	}
}

func buildSectionedDir(t *testing.T, mc *MetaCache, dir util.FullPath, snapshotTsNs int64, bounds []string) {
	t.Helper()
	if err := mc.BeginDirectoryBuild(context.Background(), dir); err != nil {
		t.Fatalf("begin build: %v", err)
	}
	if err := mc.CompleteDirectoryBuild(context.Background(), dir, snapshotTsNs, bounds); err != nil {
		t.Fatalf("complete build: %v", err)
	}
}

func applyChurn(t *testing.T, mc *MetaCache, dir, prefix string, n int, baseTsNs int64, options MetadataResponseApplyOptions) {
	t.Helper()
	for i := 0; i < n; i++ {
		resp := &filer_pb.SubscribeMetadataResponse{
			Directory: dir,
			EventNotification: &filer_pb.EventNotification{
				NewEntry: &filer_pb.Entry{
					Name: fmt.Sprintf("%s%03d", prefix, i),
					Attributes: &filer_pb.FuseAttributes{
						Crtime:   1,
						Mtime:    1,
						FileMode: 0100644,
						FileSize: 1,
					},
				},
			},
			TsNs: baseTsNs + int64(i),
		}
		if err := mc.ApplyMetadataResponse(context.Background(), resp, options); err != nil {
			t.Fatalf("apply churn %d: %v", i, err)
		}
	}
}

func TestSectionBurstInvalidatesOnlyItsSection(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})

	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)

	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("burst section should be invalidated")
	}
	if !mc.IsNameFresh(util.FullPath("/dir/zzz")) {
		t.Fatal("the other section must stay fresh")
	}
}

func TestLocalChangesDoNotInvalidateSections(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)

	applyChurn(t, mc, "/dir", "a-", 2*sectionHotThreshold, 2000, LocalMetadataResponseApplyOptions)

	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("this mount's own writes must not invalidate its cache")
	}
}

func TestSectionRefreshReconciles(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()
	mc.SetPinnedChildFn(func(entry *filer.Entry) bool {
		return entry.Name() == "b-pinned"
	})

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})

	for _, name := range []string{"b-keep", "b-vanish", "b-pinned"} {
		if err := mc.InsertEntry(context.Background(), &filer.Entry{
			FullPath: util.FullPath("/dir/" + name),
			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
		}, 0); err != nil {
			t.Fatalf("seed %s: %v", name, err)
		}
	}
	// applied ahead of the refresh snapshot; the listing must not roll it back
	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
		Directory: "/dir",
		EventNotification: &filer_pb.EventNotification{
			NewEntry: &filer_pb.Entry{
				Name:       "b-newer",
				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 9, FileMode: 0100644, FileSize: 9},
			},
		},
		TsNs: 9000,
	}, SubscriberMetadataResponseApplyOptions); err != nil {
		t.Fatalf("apply newer event: %v", err)
	}

	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
	if mc.IsNameFresh(util.FullPath("/dir/b-keep")) {
		t.Fatal("section should be stale before the refresh")
	}

	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh: &sectionRefresh{
			hi: "m",
			entries: []*filer.Entry{
				{
					FullPath: util.FullPath("/dir/b-keep"),
					Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 7},
				},
				{
					FullPath: util.FullPath("/dir/b-new"),
					Attr:     filer.Attr{Crtime: time.Unix(2, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 3},
				},
			},
			snapshotTsNs: 5000,
		},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	if !mc.IsNameFresh(util.FullPath("/dir/b-keep")) {
		t.Fatal("section should be fresh after the refresh")
	}
	entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-keep"))
	if err != nil || entry.FileSize != 7 {
		t.Fatalf("b-keep = %+v, %v; want size 7", entry, err)
	}
	if entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-new")); err != nil || entry.FileSize != 3 {
		t.Fatalf("b-new = %+v, %v; want size 3", entry, err)
	}
	if _, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-vanish")); err != filer_pb.ErrNotFound {
		t.Fatalf("b-vanish error = %v, want not found", err)
	}
	if _, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/a-000")); err != filer_pb.ErrNotFound {
		t.Fatalf("churned a-000 error = %v, want not found", err)
	}
	if entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-newer")); err != nil || entry.FileSize != 9 {
		t.Fatalf("b-newer = %+v, %v; want size 9 kept", entry, err)
	}
	if entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-pinned")); err != nil || entry == nil {
		t.Fatalf("pinned local-only entry must survive a refresh: %v", err)
	}

	// a redelivered create older than the snapshot must not resurrect b-vanish
	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
		Directory: "/dir",
		EventNotification: &filer_pb.EventNotification{
			NewEntry: &filer_pb.Entry{
				Name:       "b-vanish",
				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
			},
		},
		TsNs: 4000,
	}, SubscriberMetadataResponseApplyOptions); err != nil {
		t.Fatalf("apply stale event: %v", err)
	}
	if _, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-vanish")); err != filer_pb.ErrNotFound {
		t.Fatalf("b-vanish resurrected by a pre-snapshot event: %v", err)
	}
}

func TestSectionRefreshSplitsOvergrownSection(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)

	count := 2*dirSectionSize + 1
	entries := make([]*filer.Entry, 0, count)
	for i := 0; i < count; i++ {
		entries = append(entries, &filer.Entry{
			FullPath: util.FullPath(fmt.Sprintf("/dir/f-%05d", i)),
			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
		})
	}
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh:   &sectionRefresh{entries: entries, snapshotTsNs: 5000},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	mc.RLock()
	sl := mc.dirSections[util.FullPath("/dir")]
	bounds := append([]string(nil), sl.bounds...)
	sections := len(sl.sections)
	mc.RUnlock()
	want := []string{fmt.Sprintf("f-%05d", dirSectionSize), fmt.Sprintf("f-%05d", 2*dirSectionSize)}
	if len(bounds) != len(want) || bounds[0] != want[0] || bounds[1] != want[1] {
		t.Fatalf("bounds = %v, want %v", bounds, want)
	}
	if sections != len(bounds)+1 {
		t.Fatalf("sections = %d, want %d", sections, len(bounds)+1)
	}
}

type sectionFilerServer struct {
	filer_pb.UnimplementedSeaweedFilerServer
	mu       sync.Mutex
	names    []string // sorted; all under one directory
	snapshot int64
	requests []*filer_pb.ListEntriesRequest
}

func (s *sectionFilerServer) ListEntries(req *filer_pb.ListEntriesRequest, stream filer_pb.SeaweedFiler_ListEntriesServer) error {
	s.mu.Lock()
	s.requests = append(s.requests, req)
	names := s.names
	snapshot := s.snapshot
	s.mu.Unlock()

	sent := uint32(0)
	first := true
	for _, name := range names {
		if name < req.StartFromFileName || (name == req.StartFromFileName && !req.InclusiveStartFrom) {
			continue
		}
		resp := &filer_pb.ListEntriesResponse{Entry: &filer_pb.Entry{
			Name:       name,
			Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
		}}
		if first {
			resp.SnapshotTsNs = snapshot
			first = false
		}
		if err := stream.Send(resp); err != nil {
			return err
		}
		sent++
		if req.Limit > 0 && sent >= req.Limit {
			break
		}
	}
	return nil
}

func (s *sectionFilerServer) listRequests() []*filer_pb.ListEntriesRequest {
	s.mu.Lock()
	defer s.mu.Unlock()
	return append([]*filer_pb.ListEntriesRequest(nil), s.requests...)
}

type sectionTestFilerClient struct {
	addr string
}

func (c *sectionTestFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
	conn, err := grpc.NewClient(c.addr, grpc.WithTransportCredentials(insecure.NewCredentials()))
	if err != nil {
		return err
	}
	defer conn.Close()
	return fn(filer_pb.NewSeaweedFilerClient(conn))
}

func (c *sectionTestFilerClient) AdjustedUrl(location *filer_pb.Location) string { return location.Url }

func (c *sectionTestFilerClient) GetDataCenter() string { return "" }

func startSectionFilerServer(t *testing.T, s *sectionFilerServer) filer_pb.FilerClient {
	t.Helper()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen: %v", err)
	}
	srv := grpc.NewServer()
	filer_pb.RegisterSeaweedFilerServer(srv, s)
	go srv.Serve(listener)
	t.Cleanup(srv.Stop)
	return &sectionTestFilerClient{addr: listener.Addr().String()}
}

func TestEnsureListingFreshRefreshesFromFiler(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	server := &sectionFilerServer{snapshot: 5000}
	for i := 0; i < 1500; i++ {
		server.names = append(server.names, fmt.Sprintf("b-%04d", i))
	}
	// at and beyond the section's end, must not be applied
	server.names = append(server.names, "m", "z-1")
	client := startSectionFilerServer(t, server)

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("section should be stale before the refresh")
	}

	if err := EnsureListingFresh(context.Background(), mc, client, util.FullPath("/dir"), ""); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("section should be fresh after the refresh")
	}
	for _, name := range []string{"b-0000", "b-1499"} {
		if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name)); err != nil || entry == nil {
			t.Fatalf("%s missing after refresh: %v", name, err)
		}
	}
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/a-000")); err != filer_pb.ErrNotFound {
		t.Fatalf("churned a-000 error = %v, want not found", err)
	}
	for _, name := range []string{"m", "z-1"} {
		if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name)); err != filer_pb.ErrNotFound {
			t.Fatalf("%s at or beyond the section's end was applied: %v", name, err)
		}
	}

	requests := server.listRequests()
	if len(requests) < 2 {
		t.Fatalf("multi-page refresh made %d requests, want at least 2", len(requests))
	}
	if requests[0].SnapshotTsNs != 0 || requests[1].SnapshotTsNs != 5000 {
		t.Fatalf("snapshot not pinned across pages: %d then %d", requests[0].SnapshotTsNs, requests[1].SnapshotTsNs)
	}

	// a fresh section costs no filer calls
	if err := EnsureListingFresh(context.Background(), mc, client, util.FullPath("/dir"), ""); err != nil {
		t.Fatalf("second refresh: %v", err)
	}
	if got := len(server.listRequests()); got != len(requests) {
		t.Fatalf("fresh section still listed the filer: %d requests, was %d", got, len(requests))
	}
}

func TestEnsureListingFreshGivesUpOnOvergrownRange(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	server := &sectionFilerServer{snapshot: 5000}
	for i := 0; i <= sectionRefreshMaxEntries; i++ {
		server.names = append(server.names, fmt.Sprintf("c-%05d", i))
	}
	client := startSectionFilerServer(t, server)

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
	applyChurn(t, mc, "/dir", "x-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)

	err := EnsureListingFresh(context.Background(), mc, client, util.FullPath("/dir"), "")
	if !errors.Is(err, ErrRefreshRangeTooLarge) {
		t.Fatalf("refresh error = %v, want ErrRefreshRangeTooLarge", err)
	}
	if mc.IsNameFresh(util.FullPath("/dir/x-000")) {
		t.Fatal("an aborted refresh must leave the section stale")
	}
}

func TestSectionNoteChangeWindowExpiry(t *testing.T) {
	sl := newSectionTable(nil)
	t0 := time.Unix(100, 0)
	for i := 0; i < sectionHotThreshold-1; i++ {
		sl.noteChange(fmt.Sprintf("f-%03d", i), t0)
	}
	if !sl.isFresh("f-000") {
		t.Fatal("below the threshold the section must stay fresh")
	}

	// the burst never completed inside one window, so the count restarts
	t1 := t0.Add(sectionHotWindow + time.Second)
	for i := 0; i < sectionHotThreshold-1; i++ {
		sl.noteChange(fmt.Sprintf("f-%03d", i), t1)
	}
	if !sl.isFresh("f-000") {
		t.Fatal("an expired window must not carry its count forward")
	}

	sl.noteChange("f-999", t1)
	if sl.isFresh("f-000") {
		t.Fatal("a full burst within one window must invalidate the section")
	}
}

func TestSectionCompleteRefreshGuardsChangedTable(t *testing.T) {
	sl := newSectionTable([]string{"g", "p"})
	sl.sections[1].stale = true

	if sl.completeRefresh("g", "q", []string{"h"}, 5000) {
		t.Fatal("a range the table no longer has must be ignored")
	}
	if sl.isFresh("h") {
		t.Fatal("an ignored refresh must not mark anything fresh")
	}
	if !sl.completeRefresh("g", "p", []string{"h"}, 5000) {
		t.Fatal("the matching range must be accepted")
	}
	if !sl.isFresh("h") {
		t.Fatal("section should be fresh after the refresh")
	}
}

func TestSectionCompleteRefreshSplitsMiddleSection(t *testing.T) {
	sl := newSectionTable([]string{"g", "p"})
	sl.sections[1].stale = true

	names := make([]string, 0, 2*dirSectionSize+1)
	for i := 0; i <= 2*dirSectionSize; i++ {
		names = append(names, fmt.Sprintf("g-%05d", i))
	}
	if !sl.completeRefresh("g", "p", names, 5000) {
		t.Fatal("refresh of the middle section must be accepted")
	}

	wantBounds := []string{"g", names[dirSectionSize], names[2*dirSectionSize], "p"}
	if len(sl.bounds) != len(wantBounds) {
		t.Fatalf("bounds = %v, want %v", sl.bounds, wantBounds)
	}
	for i, b := range wantBounds {
		if sl.bounds[i] != b {
			t.Fatalf("bounds = %v, want %v", sl.bounds, wantBounds)
		}
	}
	if len(sl.sections) != len(sl.bounds)+1 {
		t.Fatalf("sections = %d, want %d", len(sl.sections), len(sl.bounds)+1)
	}
	for _, name := range []string{"a", "g-00000", names[dirSectionSize], "z"} {
		if !sl.isFresh(name) {
			t.Fatalf("%q should be fresh after the split", name)
		}
	}
	if got := sl.sectionOf(names[dirSectionSize+1]); got != 2 {
		t.Fatalf("sectionOf(%q) = %d, want 2", names[dirSectionSize+1], got)
	}
}

func TestSectionFloorFencesAbsentNames(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)

	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh:   &sectionRefresh{hi: "m", snapshotTsNs: 5000},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	// a delayed create between the directory floor and the refresh snapshot,
	// for a name neither the cache nor the listing ever held
	ghost := func(tsNs int64) {
		t.Helper()
		if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
			Directory: "/dir",
			EventNotification: &filer_pb.EventNotification{
				NewEntry: &filer_pb.Entry{
					Name:       "b-ghost",
					Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
				},
			},
			TsNs: tsNs,
		}, SubscriberMetadataResponseApplyOptions); err != nil {
			t.Fatalf("apply ghost event: %v", err)
		}
	}
	ghost(4500)
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-ghost")); err != filer_pb.ErrNotFound {
		t.Fatalf("pre-snapshot event resurrected an absent name: %v", err)
	}
	ghost(6000)
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-ghost")); err != nil {
		t.Fatalf("post-snapshot event must apply: %v", err)
	}
}

func TestUnversionedRefreshStaysStale(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)

	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh: &sectionRefresh{entries: []*filer.Entry{{
			FullPath: util.FullPath("/dir/b-gap"),
			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
		}}},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("an unversioned refresh vouches for nothing and must stay stale")
	}
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-gap")); err != nil {
		t.Fatalf("gap fill should still land: %v", err)
	}
	if ranges := mc.staleRangesAhead(util.FullPath("/dir"), ""); len(ranges) != 0 {
		t.Fatalf("an unverifiable section must not be re-listed, got %d ranges", len(ranges))
	}
}

func TestEnsureListingFreshCoversAllStaleSectionsAhead(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	server := &sectionFilerServer{snapshot: 5000, names: []string{"b-1", "n-1"}}
	client := startSectionFilerServer(t, server)

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
	applyChurn(t, mc, "/dir", "n-", sectionHotThreshold, 3000, SubscriberMetadataResponseApplyOptions)

	if err := EnsureListingFresh(context.Background(), mc, client, util.FullPath("/dir"), ""); err != nil {
		t.Fatalf("refresh: %v", err)
	}
	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) || !mc.IsNameFresh(util.FullPath("/dir/n-000")) {
		t.Fatal("one call must re-validate every stale section ahead of the start")
	}
}

func TestSectionCompleteRefreshSetsFloors(t *testing.T) {
	sl := newSectionTable([]string{"g", "p"})
	sl.sections[1].stale = true

	names := make([]string, 0, 2*dirSectionSize+1)
	for i := 0; i <= 2*dirSectionSize; i++ {
		names = append(names, fmt.Sprintf("g-%05d", i))
	}
	if !sl.completeRefresh("g", "p", names, 7000) {
		t.Fatal("refresh must be accepted")
	}
	for _, name := range []string{"g", "g-99999", names[dirSectionSize+1]} {
		if got := sl.floorOf(name); got != 7000 {
			t.Fatalf("floorOf(%q) = %d, want 7000", name, got)
		}
	}
	if got := sl.floorOf("a"); got != 0 {
		t.Fatalf("floorOf(a) = %d, want 0 for a never-refreshed section", got)
	}
}

func TestSectionFloorOutranksOlderTombstone(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)

	event := func(tsNs int64, oldEntry, newEntry *filer_pb.Entry) {
		t.Helper()
		if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
			Directory:         "/dir",
			EventNotification: &filer_pb.EventNotification{OldEntry: oldEntry, NewEntry: newEntry},
			TsNs:              tsNs,
		}, SubscriberMetadataResponseApplyOptions); err != nil {
			t.Fatalf("apply event at %d: %v", tsNs, err)
		}
	}
	pbFile := &filer_pb.Entry{
		Name:       "b-x",
		Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
	}
	event(1500, nil, pbFile)                       // create
	event(2000, &filer_pb.Entry{Name: "b-x"}, nil) // delete, leaves a tombstone at 2000

	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2500, SubscriberMetadataResponseApplyOptions)
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh:   &sectionRefresh{snapshotTsNs: 5000},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	// the floor at 5000 outranks the tombstone at 2000
	event(4000, nil, pbFile)
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-x")); err != filer_pb.ErrNotFound {
		t.Fatalf("pre-floor event applied over an older tombstone: %v", err)
	}
	event(6000, nil, pbFile)
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-x")); err != nil {
		t.Fatalf("post-floor event must apply: %v", err)
	}
}

func TestMismatchedRefreshLeavesStoreAlone(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
	if err := mc.InsertEntry(context.Background(), &filer.Entry{
		FullPath: util.FullPath("/dir/b-keep"),
		Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
	}, 0); err != nil {
		t.Fatalf("seed: %v", err)
	}
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)

	// a range the table does not have: the reconcile must not touch the store
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh: &sectionRefresh{hi: "q", snapshotTsNs: 5000, entries: []*filer.Entry{{
			FullPath: util.FullPath("/dir/b-new"),
			Attr:     filer.Attr{Crtime: time.Unix(2, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 3},
		}}},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-new")); err != filer_pb.ErrNotFound {
		t.Fatalf("mismatched refresh inserted an entry: %v", err)
	}
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-keep")); err != nil {
		t.Fatalf("mismatched refresh swept an entry: %v", err)
	}
	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("mismatched refresh must not mark anything fresh")
	}
}

func TestRefreshClearsUnversionedMarker(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()
	mc.SetPinnedChildFn(func(entry *filer.Entry) bool {
		return entry.Name() == "b-pin"
	})

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
	for _, name := range []string{"b-local", "b-pin"} {
		if err := mc.InsertEntry(context.Background(), &filer.Entry{
			FullPath: util.FullPath("/dir/" + name),
			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
		}, 0); err != nil { // versionTsNs 0 leaves the unversioned marker
			t.Fatalf("seed %s: %v", name, err)
		}
	}
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)

	refreshed := func(name string) *filer.Entry {
		return &filer.Entry{
			FullPath: util.FullPath("/dir/" + name),
			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 7},
		}
	}
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh:   &sectionRefresh{snapshotTsNs: 5000, entries: []*filer.Entry{refreshed("b-local"), refreshed("b-pin")}},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	entry, versionTsNs, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-local"))
	if err != nil || entry.FileSize != 7 {
		t.Fatalf("b-local = %+v, %v; want the snapshot's size 7", entry, err)
	}
	if versionTsNs != 5000 {
		t.Fatalf("b-local version = %d, want the floor 5000 once the marker is cleared", versionTsNs)
	}

	// a delayed event below the floor must not roll the snapshot write back
	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
		Directory: "/dir",
		EventNotification: &filer_pb.EventNotification{
			NewEntry: &filer_pb.Entry{
				Name:       "b-local",
				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 3, FileMode: 0100644, FileSize: 9},
			},
		},
		TsNs: 4000,
	}, SubscriberMetadataResponseApplyOptions); err != nil {
		t.Fatalf("apply delayed event: %v", err)
	}
	if entry, _, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-local")); err != nil || entry.FileSize != 7 {
		t.Fatalf("b-local rolled back to %+v, %v", entry, err)
	}

	// pinned local-only state is neither replaced nor unmarked
	entry, versionTsNs, err = mc.FindEntry(context.Background(), util.FullPath("/dir/b-pin"))
	if err != nil || entry.FileSize != 1 {
		t.Fatalf("b-pin = %+v, %v; want local size 1 kept", entry, err)
	}
	if versionTsNs != 0 {
		t.Fatalf("b-pin version = %d, want 0 while pinned", versionTsNs)
	}
}

func TestRefreshSkipsBuildingDirectory(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, nil)
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)

	// a rebuild is streaming: its inserts must survive a refresh's sweep
	if err := mc.BeginDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
		t.Fatalf("begin build: %v", err)
	}
	if err := mc.InsertEntry(context.Background(), &filer.Entry{
		FullPath: util.FullPath("/dir/b-built"),
		Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
	}, 0); err != nil {
		t.Fatalf("insert mid-build: %v", err)
	}

	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh:   &sectionRefresh{snapshotTsNs: 5000},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-built")); err != nil {
		t.Fatalf("refresh swept a mid-build insert: %v", err)
	}
	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("a skipped refresh must not mark the section fresh")
	}
	if err := mc.AbortDirectoryBuild(context.Background(), util.FullPath("/dir")); err != nil {
		t.Fatalf("abort build: %v", err)
	}
}

// TestSectionBorderEntries pins the border semantics: a bound-named entry
// belongs to the section starting at the bound, a refresh sweeps [lo, hi)
// inclusive of lo and exclusive of hi, and the neighboring sections' entries
// come through untouched.
func TestSectionBorderEntries(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"g", "p"})
	for _, name := range []string{"g", "h", "p"} {
		if err := mc.InsertEntry(context.Background(), &filer.Entry{
			FullPath: util.FullPath("/dir/" + name),
			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 1},
		}, 0); err != nil {
			t.Fatalf("seed %s: %v", name, err)
		}
	}
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
	applyChurn(t, mc, "/dir", "h-", sectionHotThreshold, 3000, SubscriberMetadataResponseApplyOptions)

	// the bound name is part of the section starting at it
	if mc.IsNameFresh(util.FullPath("/dir/a-000")) || mc.IsNameFresh(util.FullPath("/dir/g")) {
		t.Fatal("both churned sections should be stale")
	}

	// refreshing [ , g) sweeps its churn but must not reach the bound entry
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh:   &sectionRefresh{hi: "g", snapshotTsNs: 5000},
	}); err != nil {
		t.Fatalf("refresh section 0: %v", err)
	}
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/a-000")); err != filer_pb.ErrNotFound {
		t.Fatalf("churned a-000 should be swept: %v", err)
	}
	for _, name := range []string{"g", "h", "p"} {
		if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/"+name)); err != nil {
			t.Fatalf("%s swept by the neighboring refresh: %v", name, err)
		}
	}
	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) || mc.IsNameFresh(util.FullPath("/dir/g")) {
		t.Fatal("only the refreshed section should be fresh")
	}

	// refreshing [g, p) covers the bound entry itself and stops before p
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh: &sectionRefresh{lo: "g", hi: "p", snapshotTsNs: 5001, entries: []*filer.Entry{{
			FullPath: util.FullPath("/dir/h"),
			Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 7},
		}}},
	}); err != nil {
		t.Fatalf("refresh section 1: %v", err)
	}
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/g")); err != filer_pb.ErrNotFound {
		t.Fatalf("vanished bound entry should be swept by its own section: %v", err)
	}
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/h-000")); err != filer_pb.ErrNotFound {
		t.Fatalf("churned h-000 should be swept: %v", err)
	}
	if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/h")); err != nil || entry.FileSize != 7 {
		t.Fatalf("h = %+v, %v; want size 7", entry, err)
	}
	if _, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/p")); err != nil {
		t.Fatalf("the next bound's entry is outside the range and must survive: %v", err)
	}
}

func TestChurnBeyondBuiltEntriesLandsInEdgeSection(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})

	// names sorting past everything the build saw land in the last section
	applyChurn(t, mc, "/dir", "z-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)

	if mc.IsNameFresh(util.FullPath("/dir/z-000")) {
		t.Fatal("the tail section should be invalidated")
	}
	if !mc.IsNameFresh(util.FullPath("/dir/a")) {
		t.Fatal("the head section must stay fresh")
	}
}

func TestRenameAcrossSectionsCountsBoth(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})

	for i := 0; i < sectionHotThreshold; i++ {
		resp := &filer_pb.SubscribeMetadataResponse{
			Directory: "/dir",
			EventNotification: &filer_pb.EventNotification{
				OldEntry: &filer_pb.Entry{Name: fmt.Sprintf("a-%03d", i)},
				NewEntry: &filer_pb.Entry{
					Name:       fmt.Sprintf("n-%03d", i),
					Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 1},
				},
				NewParentPath: "/dir",
			},
			TsNs: 2000 + int64(i),
		}
		if err := mc.ApplyMetadataResponse(context.Background(), resp, SubscriberMetadataResponseApplyOptions); err != nil {
			t.Fatalf("apply rename %d: %v", i, err)
		}
	}

	if mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("the vacated side's section should be invalidated")
	}
	if mc.IsNameFresh(util.FullPath("/dir/n-000")) {
		t.Fatal("the landing side's section should be invalidated")
	}
}

// A refresh sweeps the names its listing did not return. One recorded at
// exactly the snapshot has its write reflected but not its removal, so reading
// the gate as a write left it cached for good.
func TestSectionRefreshSweepsANameRecordedAtTheSnapshot(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/dir": true})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})

	const snapshotTsNs = 5000
	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
		Directory: "/dir",
		EventNotification: &filer_pb.EventNotification{
			NewEntry: &filer_pb.Entry{
				Name:       "b-vanish",
				Attributes: &filer_pb.FuseAttributes{Crtime: 1, Mtime: 1, FileMode: 0100644, FileSize: 3},
			},
		},
		TsNs: snapshotTsNs,
	}, SubscriberMetadataResponseApplyOptions); err != nil {
		t.Fatalf("seed at the snapshot version: %v", err)
	}

	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh: &sectionRefresh{
			hi: "m",
			entries: []*filer.Entry{
				{
					FullPath: util.FullPath("/dir/b-keep"),
					Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 7},
				},
			},
			snapshotTsNs: snapshotTsNs,
		},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	if entry, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b-vanish")); err == nil && entry != nil {
		t.Error("a name the refresh did not list is still cached")
	}
}

// TestSectionRefreshParksHardLinkBlobContent checks that a stale snapshot
// install of an uncached hard-link sibling keeps the name but parks the
// group's shared blob content, rather than rolling every sibling back to
// the snapshot's content.
func TestSectionRefreshParksHardLinkBlobContent(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/dir": true})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})

	hardLinkId := filer.HardLinkId([]byte{1, 2, 3, 4})
	const writeTsNs = 6000
	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
		Directory: "/dir",
		EventNotification: &filer_pb.EventNotification{
			NewEntry: &filer_pb.Entry{
				Name:            "a",
				HardLinkId:      hardLinkId,
				HardLinkCounter: 2,
				Attributes:      &filer_pb.FuseAttributes{Crtime: 1, Mtime: 9, FileMode: 0100644, FileSize: 99},
			},
		},
		TsNs: writeTsNs,
	}, SubscriberMetadataResponseApplyOptions); err != nil {
		t.Fatalf("seed sibling at %d: %v", writeTsNs, err)
	}

	// The refresh's snapshot predates the write: sibling "b" is live on the
	// filer but its listed content is older than the group's shared blob.
	const snapshotTsNs = 5000
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh: &sectionRefresh{
			hi: "m",
			entries: []*filer.Entry{
				{
					FullPath:        util.FullPath("/dir/b"),
					Attr:            filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 3},
					HardLinkId:      hardLinkId,
					HardLinkCounter: 2,
				},
			},
			snapshotTsNs: snapshotTsNs,
		},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	b, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b"))
	if err != nil || b == nil {
		t.Fatalf("the live link is missing after the refresh: %v", err)
	}
	if b.FileSize != 99 {
		t.Errorf("sibling installed the snapshot's stale content: FileSize=%d, want 99", b.FileSize)
	}
	a, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/a"))
	if err != nil || a == nil {
		t.Fatalf("seeded sibling missing: %v", err)
	}
	if a.FileSize != 99 {
		t.Errorf("the shared blob was rolled back: FileSize=%d, want 99", a.FileSize)
	}
}

// TestSectionRefreshReplacesReusedNameOccupant checks that a cached entry
// sharing only the listed name — a path since recreated as a hard-link
// member — is replaced by the group's parked content rather than kept,
// which would serve the stale occupant under a fresh section.
func TestSectionRefreshReplacesReusedNameOccupant(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/dir": true})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})

	if err := mc.InsertEntry(context.Background(), &filer.Entry{
		FullPath: util.FullPath("/dir/b"),
		Attr:     filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 7},
	}, 4000); err != nil {
		t.Fatalf("seed occupant: %v", err)
	}

	hardLinkId := filer.HardLinkId([]byte{1, 2, 3, 4})
	const writeTsNs = 6000
	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
		Directory: "/dir",
		EventNotification: &filer_pb.EventNotification{
			NewEntry: &filer_pb.Entry{
				Name:            "a",
				HardLinkId:      hardLinkId,
				HardLinkCounter: 2,
				Attributes:      &filer_pb.FuseAttributes{Crtime: 1, Mtime: 9, FileMode: 0100644, FileSize: 99},
			},
		},
		TsNs: writeTsNs,
	}, SubscriberMetadataResponseApplyOptions); err != nil {
		t.Fatalf("seed sibling at %d: %v", writeTsNs, err)
	}

	const snapshotTsNs = 5000
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh: &sectionRefresh{
			hi: "m",
			entries: []*filer.Entry{
				{
					FullPath:        util.FullPath("/dir/b"),
					Attr:            filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 3},
					HardLinkId:      hardLinkId,
					HardLinkCounter: 2,
				},
			},
			snapshotTsNs: snapshotTsNs,
		},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	b, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b"))
	if err != nil || b == nil {
		t.Fatalf("the live link is missing after the refresh: %v", err)
	}
	if string(b.HardLinkId) != string(hardLinkId) {
		t.Errorf("the stale occupant was kept: HardLinkId=%x, want %x", b.HardLinkId, hardLinkId)
	}
	if b.FileSize != 99 {
		t.Errorf("reused name did not get the parked content: FileSize=%d, want 99", b.FileSize)
	}
}

// TestSectionRefreshKeepsOccupantGroupCounter covers a reused name whose
// cached occupant's shared blob already reflects the filer-side unlink: a
// sibling write folded the counter down while the occupant's entry stayed
// cached. Replacing that occupant must not decrement the group again, or
// the counter reaches zero and the shared blob live siblings read is lost.
func TestSectionRefreshKeepsOccupantGroupCounter(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{"/": true, "/dir": true})
	defer mc.Shutdown()

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})

	oldGroup := filer.HardLinkId([]byte{5, 6, 7, 8})
	for _, name := range []string{"b", "q"} {
		if err := mc.InsertEntry(context.Background(), &filer.Entry{
			FullPath:        util.FullPath("/dir/" + name),
			Attr:            filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(1, 0), Mode: 0100644, FileSize: 11},
			HardLinkId:      oldGroup,
			HardLinkCounter: 2,
		}, 2000); err != nil {
			t.Fatalf("seed %s: %v", name, err)
		}
	}
	// A post-unlink write to the live sibling folds the occupant's removal
	// into the shared counter while the occupant's entry is still cached.
	if err := mc.InsertEntry(context.Background(), &filer.Entry{
		FullPath:        util.FullPath("/dir/q"),
		Attr:            filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(3, 0), Mode: 0100644, FileSize: 22},
		HardLinkId:      oldGroup,
		HardLinkCounter: 1,
	}, 3000); err != nil {
		t.Fatalf("reseed q: %v", err)
	}

	hardLinkId := filer.HardLinkId([]byte{1, 2, 3, 4})
	const writeTsNs = 6000
	if err := mc.ApplyMetadataResponse(context.Background(), &filer_pb.SubscribeMetadataResponse{
		Directory: "/dir",
		EventNotification: &filer_pb.EventNotification{
			NewEntry: &filer_pb.Entry{
				Name:            "a",
				HardLinkId:      hardLinkId,
				HardLinkCounter: 2,
				Attributes:      &filer_pb.FuseAttributes{Crtime: 1, Mtime: 9, FileMode: 0100644, FileSize: 99},
			},
		},
		TsNs: writeTsNs,
	}, SubscriberMetadataResponseApplyOptions); err != nil {
		t.Fatalf("seed sibling at %d: %v", writeTsNs, err)
	}

	const snapshotTsNs = 5000
	if err := mc.enqueueAndWait(context.Background(), metadataApplyRequest{
		kind:      metadataSectionRefresh,
		buildPath: util.FullPath("/dir"),
		refresh: &sectionRefresh{
			hi: "m",
			entries: []*filer.Entry{
				{
					FullPath:        util.FullPath("/dir/b"),
					Attr:            filer.Attr{Crtime: time.Unix(1, 0), Mtime: time.Unix(2, 0), Mode: 0100644, FileSize: 3},
					HardLinkId:      hardLinkId,
					HardLinkCounter: 2,
				},
			},
			snapshotTsNs: snapshotTsNs,
		},
	}); err != nil {
		t.Fatalf("refresh: %v", err)
	}

	b, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/b"))
	if err != nil || b == nil {
		t.Fatalf("the live link is missing after the refresh: %v", err)
	}
	if string(b.HardLinkId) != string(hardLinkId) {
		t.Errorf("the stale occupant was kept: HardLinkId=%x, want %x", b.HardLinkId, hardLinkId)
	}

	blob, err := mc.localStore.KvGet(context.Background(), oldGroup)
	if err != nil {
		t.Fatalf("the live sibling's shared blob was deleted: %v", err)
	}
	shared := &filer.Entry{}
	if err := shared.DecodeAttributesAndChunks(blob); err != nil {
		t.Fatalf("decode shared blob: %v", err)
	}
	if shared.HardLinkCounter != 1 {
		t.Errorf("the occupant's old group was decremented again: counter=%d, want 1", shared.HardLinkCounter)
	}
	q, _, err := mc.FindEntry(context.Background(), util.FullPath("/dir/q"))
	if err != nil || q == nil {
		t.Fatalf("the live sibling is unreadable after the refresh: %v", err)
	}
	if q.FileSize != 22 {
		t.Errorf("the live sibling's content was rolled back: FileSize=%d, want 22", q.FileSize)
	}
}

// TestEnsureNameFreshRefreshesOnlyItsSection checks that a name-scoped
// refresh re-lists just the section holding that name and leaves other stale
// sections untouched.
func TestEnsureNameFreshRefreshesOnlyItsSection(t *testing.T) {
	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
		"/":    true,
		"/dir": true,
	})
	defer mc.Shutdown()

	server := &sectionFilerServer{snapshot: 5000}
	server.names = append(server.names, "a-000", "b-000", "m", "z-000")
	client := startSectionFilerServer(t, server)

	buildSectionedDir(t, mc, util.FullPath("/dir"), 1000, []string{"m"})
	applyChurn(t, mc, "/dir", "a-", sectionHotThreshold, 2000, SubscriberMetadataResponseApplyOptions)
	applyChurn(t, mc, "/dir", "z-", sectionHotThreshold, 3000, SubscriberMetadataResponseApplyOptions)
	if mc.IsNameFresh(util.FullPath("/dir/a-000")) || mc.IsNameFresh(util.FullPath("/dir/z-000")) {
		t.Fatal("both sections should be stale before the refresh")
	}

	if err := EnsureNameFresh(context.Background(), mc, client, util.FullPath("/dir"), "a-000"); err != nil {
		t.Fatalf("refresh: %v", err)
	}
	if !mc.IsNameFresh(util.FullPath("/dir/a-000")) {
		t.Fatal("the section holding the name should be fresh after the refresh")
	}
	if mc.IsNameFresh(util.FullPath("/dir/z-000")) {
		t.Fatal("the other stale section must not be refreshed by a name outside it")
	}

	requests := server.listRequests()
	if len(requests) != 1 {
		t.Fatalf("name refresh made %d list requests, want 1", len(requests))
	}
	if requests[0].StartFromFileName != "" || requests[0].InclusiveStartFrom {
		t.Fatalf("refresh should list the section from its start: StartFromFileName=%q inclusive=%v",
			requests[0].StartFromFileName, requests[0].InclusiveStartFrom)
	}

	if err := EnsureNameFresh(context.Background(), mc, client, util.FullPath("/dir"), "z-000"); err != nil {
		t.Fatalf("second refresh: %v", err)
	}
	if !mc.IsNameFresh(util.FullPath("/dir/z-000")) {
		t.Fatal("the second section should be fresh after its own refresh")
	}
}