SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
filer: create parent directories for lazily fetched remote entries (#11727)
gflatters103328e70fce
diff --git a/weed/filer/filer.go b/weed/filer/filer.goindex 0a5d3540be..249ad4acd7 100644--- a/weed/filer/filer.go+++ b/weed/filer/filer.go@@ -890,7 +890,7 @@ func (f *Filer) DeleteEmptyParentDirectories(ctx context.Context, dirPath util.Fglog.V(2).InfofCtx(ctx, "DeleteEmptyParentDirectories: deleting empty directory %s", dirPath)parentDir, _ := dirPath.DirAndName()if dirEntry, findErr := f.FindEntry(ctx, dirPath); findErr == nil {- if delErr := f.doDeleteEntryMetaAndData(ctx, dirEntry, false, false, nil); delErr == nil {+ if delErr := f.doDeleteEntryMetaAndData(f.emptyFolderDeleteContext(ctx, dirPath), dirEntry, false, false, nil); delErr == nil {// Successfully deleted, continue checking upwardsf.DeleteEmptyParentDirectories(ctx, util.FullPath(parentDir), stopAtPath)} else {diff --git a/weed/filer/filer_delete_entry.go b/weed/filer/filer_delete_entry.goindex 5aca5566a3..bec4d7f687 100644--- a/weed/filer/filer_delete_entry.go+++ b/weed/filer/filer_delete_entry.go@@ -80,6 +80,27 @@ func keepsRemoteObject(ctx context.Context) bool {return ctx.Value(keepRemoteObjectKey{}) != nil}+// emptyFolderDeleteContext makes deleting a folder that is empty in the filer+// metadata-only when it is under a remote mount. A mount is populated lazily,+// so the folder can still hold remote objects never read, and removing its+// remote directory deletes every object under that prefix.+func (f *Filer) emptyFolderDeleteContext(ctx context.Context, dirPath util.FullPath) context.Context {+ if f.IsUnderRemoteMount(dirPath) {+ return WithKeepRemoteObject(ctx)+ }+ return ctx+}++// IsUnderRemoteMount reports whether p is a remote mount directory or lies under one.+func (f *Filer) IsUnderRemoteMount(p util.FullPath) bool {+ if f.RemoteStorage == nil {+ return false+ }+ // Mount rules are keyed with a trailing slash, so this also matches the mount directory itself.+ _, remoteLoc := f.RemoteStorage.FindMountDirectory(util.FullPath(string(p) + "/"))+ return remoteLoc != nil+}+func deleteEventEntry(ctx context.Context, entry *Entry) *Entry {if !keepsRemoteObject(ctx) {return entry@@ -108,6 +129,9 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRif ifNotModifiedAfter > 0 && entry.Attr.Mtime.Unix() > ifNotModifiedAfter {return nil}+ if !isRecursive && entry.IsDirectory() {+ ctx = f.emptyFolderDeleteContext(ctx, p)+ }isDeleteCollection := f.IsBucket(entry)collectionName := ""if isDeleteCollection {@@ -170,7 +194,7 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isRfunc (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entry, isRecursive, ignoreRecursiveError, shouldDeleteChunks, isDeletingBucket, isFromOtherCluster bool, signatures []int32, onHardLinkIdsFn OnHardLinkIdsFunc) (err error) {var dirTombstoneTs int64- if isRecursive {+ if isRecursive && !keepsRemoteObject(ctx) {// Tombstone the directory before its children: when the store drops// the subtree without listing it, or a child error aborts the walk,// the ancestor tombstone still covers every descendant.@@ -201,7 +225,9 @@ func (f *Filer) doBatchDeleteFolderMetaAndData(ctx context.Context, entry *Entryfor _, sub := range entries {lastFileName = sub.Name()- f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano())+ if !keepsRemoteObject(ctx) {+ f.noteRemoteDeletion(sub.FullPath, sub.IsDirectory(), time.Now().UnixNano())+ }if sub.IsDirectory() {subIsDeletingBucket := f.IsBucket(sub)err = f.doBatchDeleteFolderMetaAndData(ctx, sub, isRecursive, ignoreRecursiveError, shouldDeleteChunks, subIsDeletingBucket, isFromOtherCluster, nil, onHardLinkIdsFn)@@ -264,7 +290,10 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou}}- f.noteRemoteDeletion(entry.FullPath, entry.IsDirectory(), time.Now().UnixNano())+ // A metadata-only delete owes the remote nothing, so lazy reads stay open.+ if !keepsRemoteObject(ctx) {+ f.noteRemoteDeletion(entry.FullPath, entry.IsDirectory(), time.Now().UnixNano())+ }if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {return fmt.Errorf("filer store delete: %w", storeDeletionErr)diff --git a/weed/filer/filer_lazy_remote.go b/weed/filer/filer_lazy_remote.goindex 09110adfae..036ac5f5dc 100644--- a/weed/filer/filer_lazy_remote.go+++ b/weed/filer/filer_lazy_remote.go@@ -18,11 +18,15 @@ import (type lazyFetchContextKey struct{}+// lazyParentsContextKey marks a FindEntry issued while materializing missing+// parents: that fetch's own parent walk would only re-stat the same gaps.+type lazyParentsContextKey struct{}+// maybeLazyFetchFromRemote is called by FindEntry when the store returns no// entry for p. If p is under a remote-storage mount, it stats the remote// object, builds a filer Entry from the result, and persists it via-// CreateEntry with SkipCheckParentDirectory so phantom parent directories-// under the mount are not required.+// CreateEntry together with its missing parent directories: remote.uncache+// and cache eviction find entries by walking down from the mount directory.//// On a CreateEntry failure after a successful StatFile the in-memory entry is// still returned (availability over consistency); the singleflight key is@@ -127,7 +131,10 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (glog.V(2).InfofCtx(ctx, "maybeLazyFetchFromRemote: %s deleted during remote stat", p)return lazyFetchResult{nil}, nil}- saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, true, f.MaxFilenameLength)+ if ctx.Value(lazyParentsContextKey{}) == nil {+ f.fetchMissingRemoteParents(persistBaseCtx, mountDir, p)+ }+ saveErr := f.CreateEntry(persistCtx, entry, nil, false, false, nil, false, f.MaxFilenameLength)if saveErr != nil {glog.Warningf("maybeLazyFetchFromRemote: failed to persist filer entry for %s: %v", p, saveErr)f.lazyFetchGroup.Forget(key)@@ -157,6 +164,25 @@ func (f *Filer) maybeLazyFetchFromRemote(ctx context.Context, p util.FullPath) (return result.entry, nil}+// fetchMissingRemoteParents looks up p's missing parents under mountDir, top+// down, through lazy fetch. A remote object stored at a parent's key is then+// materialized first, so creating p promotes it to a prefix object instead of+// hiding it behind a plain directory. Each missing parent costs one remote+// stat, once per directory.+func (f *Filer) fetchMissingRemoteParents(ctx context.Context, mountDir, p util.FullPath) {+ var missing []util.FullPath+ for dir, _ := p.DirAndName(); len(dir) > len(mountDir); dir, _ = util.FullPath(dir).DirAndName() {+ if _, err := f.Store.FindEntry(ctx, util.FullPath(dir)); !errors.Is(err, filer_pb.ErrNotFound) {+ break+ }+ missing = append(missing, util.FullPath(dir))+ }+ parentCtx := context.WithValue(ctx, lazyParentsContextKey{}, true)+ for i := len(missing) - 1; i >= 0; i-- {+ f.FindEntry(parentCtx, missing[i])+ }+}+// retractLazyRemoteEntry deletes the entry at entry.FullPath only when it is// still the entry a lazy remote read just materialized — a concurrent write// may have replaced it, and deleting by path alone would take that write down.diff --git a/weed/filer/filer_lazy_remote_listing.go b/weed/filer/filer_lazy_remote_listing.goindex d692908a81..52b43981fe 100644--- a/weed/filer/filer_lazy_remote_listing.go+++ b/weed/filer/filer_lazy_remote_listing.go@@ -103,6 +103,10 @@ func (f *Filer) maybeLazyListFromRemote(ctx context.Context, p util.FullPath) {persistCtx := context.WithValue(context.WithoutCancel(ctx), lazyListContextKey{}, true)persistCtx = context.WithValue(persistCtx, lazyFetchContextKey{}, true)+ // Entries listed under p must stay reachable by a walk from the mount+ // directory, the same reason the lazy fetch fills in missing parents.+ f.fetchMissingRemoteParents(context.WithoutCancel(ctx), mountDir, p)+listErr := client.ListDirectory(persistCtx, objectLoc, func(dir string, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {childPath := p.Child(name)@@ -215,7 +219,7 @@ func (f *Filer) updateDirectoryListingSyncedAt(ctx context.Context, p util.FullPdirEntry.Extended = make(map[string][]byte)}dirEntry.Extended[xattrRemoteListingSyncedAt] = []byte(fmt.Sprintf("%d", syncTime.Unix()))- if saveErr := f.CreateEntry(ctx, dirEntry, nil, false, false, nil, true, f.MaxFilenameLength); saveErr != nil {+ if saveErr := f.CreateEntry(ctx, dirEntry, nil, false, false, nil, false, f.MaxFilenameLength); saveErr != nil {glog.Warningf("maybeLazyListFromRemote: create dir synced_at for %s: %v", p, saveErr)}returndiff --git a/weed/filer/filer_lazy_remote_test.go b/weed/filer/filer_lazy_remote_test.goindex 7a867fcc6d..c56934c6f9 100644--- a/weed/filer/filer_lazy_remote_test.go+++ b/weed/filer/filer_lazy_remote_test.go@@ -17,6 +17,7 @@ import ("github.com/seaweedfs/seaweedfs/weed/pb/filer_pb""github.com/seaweedfs/seaweedfs/weed/pb/remote_pb""github.com/seaweedfs/seaweedfs/weed/remote_storage"+ "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants""github.com/seaweedfs/seaweedfs/weed/util""github.com/seaweedfs/seaweedfs/weed/util/log_buffer""github.com/seaweedfs/seaweedfs/weed/wdclient"@@ -35,6 +36,7 @@ type stubFilerStore struct {kv map[string][]byteinsertErr errorfindErr error+ findErrByPath map[string]errorkvPutErr errorkvGetErr errorkvDeleteErr error@@ -200,6 +202,9 @@ func (s *stubFilerStore) FindEntry(_ context.Context, p util.FullPath) (*Entry,if s.findErr != nil {return nil, s.findErr}+ if err := s.findErrByPath[string(p)]; err != nil {+ return nil, err+ }if e, ok := s.entries[string(p)]; ok {return e, nil}@@ -340,6 +345,11 @@ func putConfigEntry(store *stubFilerStore, path string, content []byte) {}}+// putDirEntry stores a plain directory entry, so a lazy fetch below it finds its parent in place.+func putDirEntry(store *stubFilerStore, path string) {+ store.entries[path] = &Entry{FullPath: util.FullPath(path), Attr: Attr{Mode: os.ModeDir | 0755}}+}+// --- tests ---func TestMaybeLazyFetchFromRemote_HitsRemoteAndPersists(t *testing.T) {@@ -375,6 +385,199 @@ func TestMaybeLazyFetchFromRemote_HitsRemoteAndPersists(t *testing.T) {assert.Equal(t, []byte("zstd"), stored.Extended["Content-Encoding"])}+// pathStatRemoteClient stats only the objects it holds, keyed by remote path.+type pathStatRemoteClient struct {+ stubRemoteClient+ objects map[string]*filer_pb.RemoteEntry+ statCalls []string+}++func (c *pathStatRemoteClient) StatFile(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {+ c.statCalls = append(c.statCalls, loc.Path)+ if remoteEntry, found := c.objects[loc.Path]; found {+ return remoteEntry, nil+ }+ return nil, remote_storage.ErrRemoteObjectNotFound+}++func newPathStatFiler(t *testing.T, storageType string, client remote_storage.RemoteStorageClient) (*Filer, *stubFilerStore) {+ t.Helper()+ t.Cleanup(registerStubMaker(t, storageType, client))+ conf := &remote_pb.RemoteConf{Name: "mystore", Type: storageType}+ rs := NewFilerRemoteStorage()+ rs.storageNameToConf[conf.Name] = conf+ rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{+ Name: "mystore",+ Bucket: "mybucket",+ Path: "/",+ })+ store := newStubFilerStore()+ return newTestFiler(t, store, rs), store+}++func TestMaybeLazyFetchFromRemote_PersistsParentDirectories(t *testing.T) {+ stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{+ "/a/b/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234},+ "/a/b/other.bin": {RemoteMtime: 1700000000, RemoteSize: 5678},+ }}+ f, store := newPathStatFiler(t, "stub_lazy_parents", stub)++ const filePath = "/buckets/mybucket/a/b/file.bin"+ entry, err := f.maybeLazyFetchFromRemote(context.Background(), filePath)+ require.NoError(t, err)+ require.NotNil(t, entry)+ assert.Equal(t, []string{"/a/b/file.bin", "/a", "/a/b"}, stub.statCalls, "each missing parent is stat'ed once")++ for _, dir := range []string{"/buckets/mybucket/a", "/buckets/mybucket/a/b"} {+ dirEntry := store.getEntry(dir)+ require.NotNil(t, dirEntry, "parent %s not persisted", dir)+ assert.True(t, dirEntry.IsDirectory(), "%s should be a directory", dir)+ assert.Nil(t, dirEntry.Remote, "%s holds no remote object", dir)+ }++ // A sibling finds its parents in place and stats only itself.+ stub.statCalls = nil+ _, err = f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/other.bin")+ require.NoError(t, err)+ assert.Equal(t, []string{"/a/b/other.bin"}, stub.statCalls)++ // Once the content is cached, eviction walks down from the mount and must reach it.+ stored := store.getEntry(filePath)+ require.NotNil(t, stored)+ stored.Chunks = []*filer_pb.FileChunk{{FileId: "1,01637037d6", Size: 1234}}+ stored.Remote.LastLocalSyncTsNs = time.Now().UnixNano()+ require.NoError(t, store.UpdateEntry(context.Background(), stored))++ evictable := f.ListEvictableRemoteEntries(context.Background(), []util.FullPath{"/buckets/mybucket"}, 0)+ require.Len(t, evictable, 1)+ assert.Equal(t, util.FullPath(filePath), evictable[0].FullPath)+}++func TestMaybeLazyFetchFromRemote_ParentLookupStopsAtStoreError(t *testing.T) {+ stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{+ "/a/b/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234},+ }}+ f, store := newPathStatFiler(t, "stub_lazy_parent_store_error", stub)+ store.findErrByPath = map[string]error{"/buckets/mybucket/a/b": errors.New("store unavailable")}++ _, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/file.bin")+ require.NoError(t, err)+ assert.Equal(t, []string{"/a/b/file.bin"}, stub.statCalls, "a parent the store cannot read is not treated as missing")+}++func TestMaybeLazyFetchFromRemote_PromotesRemoteObjectAtParentKey(t *testing.T) {+ stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{+ "/reports": {RemoteMtime: 1700000000, RemoteSize: 10},+ "/reports/2026.csv": {RemoteMtime: 1700000000, RemoteSize: 20},+ }}+ f, store := newPathStatFiler(t, "stub_lazy_prefix_object", stub)++ _, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/reports/2026.csv")+ require.NoError(t, err)++ parent := store.getEntry("/buckets/mybucket/reports")+ require.NotNil(t, parent)+ assert.True(t, parent.IsDirectory())+ assert.Equal(t, []byte("true"), parent.Extended[s3_constants.SeaweedFSPrefixObject], "the remote object at the parent key must stay readable")+ require.NotNil(t, parent.Remote)+ assert.Equal(t, int64(10), parent.Remote.RemoteSize)+}++func TestMaybeLazyFetchFromRemote_StatsEachParentOnce(t *testing.T) {+ stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{+ "/a/b/c/file.bin": {RemoteMtime: 1700000000, RemoteSize: 1234},+ }}+ f, _ := newPathStatFiler(t, "stub_lazy_parent_stats", stub)++ _, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/a/b/c/file.bin")+ require.NoError(t, err)+ assert.Equal(t, []string{"/a/b/c/file.bin", "/a", "/a/b", "/a/b/c"}, stub.statCalls,+ "a parent still missing after its own stat must not be re-stat'ed by the next lookup")+}++func TestMaybeLazyListFromRemote_PersistsParentDirectories(t *testing.T) {+ stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{}}+ stub.listDirFn = func(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) error {+ return visitFn("/", "file.bin", false, &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 1234})+ }+ f, store := newPathStatFiler(t, "stub_lazy_list_parents", stub)+ f.RemoteStorage.mapDirectoryToRemoteStorage("/buckets/listmount", &remote_pb.RemoteStorageLocation{+ Name: "mystore",+ Bucket: "listmount",+ Path: "/",+ ListingCacheTtlSeconds: 300,+ })++ f.maybeLazyListFromRemote(context.Background(), util.FullPath("/buckets/listmount/a/b"))++ assert.Equal(t, []string{"/a"}, stub.statCalls, "each missing parent is stat'ed once")+ for _, dir := range []string{"/buckets/listmount", "/buckets/listmount/a", "/buckets/listmount/a/b"} {+ dirEntry := store.getEntry(dir)+ require.NotNil(t, dirEntry, "%s not persisted", dir)+ assert.True(t, dirEntry.IsDirectory(), "%s should be a directory", dir)+ }++ child := store.getEntry("/buckets/listmount/a/b/file.bin")+ require.NotNil(t, child)+ child.Chunks = []*filer_pb.FileChunk{{FileId: "1,01637037d6", Size: 1234}}+ child.Remote.LastLocalSyncTsNs = time.Now().UnixNano()+ require.NoError(t, store.UpdateEntry(context.Background(), child))++ evictable := f.ListEvictableRemoteEntries(context.Background(), []util.FullPath{"/buckets/listmount"}, 0)+ require.Len(t, evictable, 1)+ assert.Equal(t, util.FullPath("/buckets/listmount/a/b/file.bin"), evictable[0].FullPath)+}++func TestDeleteEntryMetaAndData_EmptyFolderUnderMountKeepsRemotePrefix(t *testing.T) {+ stub := &pathStatRemoteClient{objects: map[string]*filer_pb.RemoteEntry{+ "/dir/unread.bin": {RemoteMtime: 1700000000, RemoteSize: 1},+ }}+ f, store := newPathStatFiler(t, "stub_lazy_empty_folder", stub)+ dirPath := util.FullPath("/buckets/mybucket/dir")+ putDirEntry(store, string(dirPath))++ require.NoError(t, f.DeleteEntryMetaAndData(context.Background(), dirPath, false, false, false, false, nil, 0))++ assert.Empty(t, stub.removeCalls, "an empty local folder must not remove its remote prefix")+ assert.Nil(t, store.getEntry(string(dirPath)))+ entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/unread.bin")+ require.NoError(t, err)+ assert.NotNil(t, entry, "a metadata-only delete must not block lazy reads under the folder")+}++func TestDeleteEmptyParentDirectories_UnderMountKeepsRemotePrefix(t *testing.T) {+ stub := &pathStatRemoteClient{}+ f, store := newPathStatFiler(t, "stub_lazy_expired_parents", stub)+ f.DirBucketsPath = "/buckets"+ putDirEntry(store, "/buckets/mybucket/a")+ putDirEntry(store, "/buckets/mybucket/a/b")++ f.DeleteEmptyParentDirectories(context.Background(), "/buckets/mybucket/a/b", "/buckets")++ assert.Empty(t, stub.removeCalls, "an empty local folder must not remove its remote prefix")+ assert.Nil(t, store.getEntry("/buckets/mybucket/a/b"))+ assert.Nil(t, store.getEntry("/buckets/mybucket/a"))+}++func TestIsUnderRemoteMount(t *testing.T) {+ rs := NewFilerRemoteStorage()+ rs.mapDirectoryToRemoteStorage("/buckets/mirror", &remote_pb.RemoteStorageLocation{+ Name: "mystore",+ Bucket: "mirror",+ Path: "/",+ })+ f := newTestFiler(t, newStubFilerStore(), rs)++ for path, want := range map[util.FullPath]bool{+ "/buckets/mirror": true,+ "/buckets/mirror/a/b": true,+ "/buckets/mirror2/a": false,+ "/buckets/local/a": false,+ } {+ assert.Equal(t, want, f.IsUnderRemoteMount(path), path)+ }+}+func TestMaybeLazyFetchFromRemote_NotUnderMount(t *testing.T) {rs := NewFilerRemoteStorage()store := newStubFilerStore()diff --git a/weed/filer/filer_remote_tombstone.go b/weed/filer/filer_remote_tombstone.goindex bf7b4b187f..2f4c7764d8 100644--- a/weed/filer/filer_remote_tombstone.go+++ b/weed/filer/filer_remote_tombstone.go@@ -407,7 +407,7 @@ func (f *Filer) onRemoteDeletionEvents(event *filer_pb.SubscribeMetadataResponseif message == nil {return}- if message.OldEntry != nil {+ if message.OldEntry != nil && !IsMetadataOnlyDelete(message.OldEntry) {sourcePath := filer_pb.MetadataEventSourceFullPath(event)if message.NewEntry == nil || sourcePath != filer_pb.MetadataEventTargetFullPath(event) {if f.RemoteStorage != nil && f.remoteTombstones != nil {diff --git a/weed/filer/filer_remote_tombstone_test.go b/weed/filer/filer_remote_tombstone_test.goindex 3d989d4c37..6048ae9575 100644--- a/weed/filer/filer_remote_tombstone_test.go+++ b/weed/filer/filer_remote_tombstone_test.go@@ -187,6 +187,7 @@ func TestMaybeLazyFetchFromRemote_SyncOffsetReleasesTombstone(t *testing.T) {},}f, store := newMountedTestFiler(t, storageType, stub, 0)+ putDirEntry(store, "/buckets/mybucket/dir")filePath := util.FullPath("/buckets/mybucket/dir/a.txt")deleteTsNs := time.Now().UnixNano()@@ -222,6 +223,7 @@ func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) {},}f, store := newMountedTestFiler(t, storageType, stub, 0)+ putDirEntry(store, "/buckets/mybucket/dir")filePath := util.FullPath("/buckets/mybucket/dir/a.txt")store.entries[string(filePath)] = &Entry{@@ -257,6 +259,35 @@ func TestDeleteEntryMetaAndData_TombstonesPath(t *testing.T) {assert.Equal(t, 1, stub.statCalls)}+func TestOnMetadataChangeEvent_MetadataOnlyDeleteSkipsTombstone(t *testing.T) {+ const storageType = "stub_tomb_metadata_only"+ stub := &countingRemoteClient{+ stubRemoteClient: stubRemoteClient{+ statResult: &filer_pb.RemoteEntry{RemoteMtime: 1700000000, RemoteSize: 11},+ },+ }+ f, store := newMountedTestFiler(t, storageType, stub, 0)+ putDirEntry(store, "/buckets/mybucket/dir")++ f.onMetadataChangeEvent(&filer_pb.SubscribeMetadataResponse{+ Directory: "/buckets/mybucket",+ TsNs: time.Now().UnixNano(),+ EventNotification: &filer_pb.EventNotification{+ OldEntry: &filer_pb.Entry{+ Name: "dir",+ IsDirectory: true,+ Extended: map[string][]byte{ExtKeepRemoteObjectKey: []byte("true")},+ },+ },+ })++ // nothing is owed to the remote, so objects under the folder stay readable+ entry, err := f.maybeLazyFetchFromRemote(context.Background(), "/buckets/mybucket/dir/a.txt")+ require.NoError(t, err)+ require.NotNil(t, entry)+ assert.Equal(t, 1, stub.statCalls)+}+func TestOnMetadataChangeEvent_PeerDeleteTombstones(t *testing.T) {const storageType = "stub_tomb_peer"stub := &countingRemoteClient{