SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
package empty_folder_cleanup
import (
"context"
"errors"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type mockFilerOps struct {
countFn func(path util.FullPath) (int, error)
deleteFn func(path util.FullPath) error
attrsFn func(path util.FullPath) (map[string][]byte, error)
isDirKeyObjFn func(path util.FullPath) (bool, error)
ensureDirFn func(path util.FullPath, attrs DirectoryAttributes) error
dirAttrsFn func(path util.FullPath) (DirectoryAttributes, error)
}
func (m *mockFilerOps) EnsureDirectoryEntry(_ context.Context, p util.FullPath, attrs DirectoryAttributes) error {
if m.ensureDirFn == nil {
return nil
}
return m.ensureDirFn(p, attrs)
}
func (m *mockFilerOps) DirectoryAttributes(_ context.Context, p util.FullPath) (DirectoryAttributes, error) {
if m.dirAttrsFn == nil {
return DirectoryAttributes{Mode: 0755}, nil
}
return m.dirAttrsFn(p)
}
func (m *mockFilerOps) CountDirectoryEntries(_ context.Context, dirPath util.FullPath, _ int) (int, error) {
if m.countFn == nil {
return 0, nil
}
return m.countFn(dirPath)
}
func (m *mockFilerOps) DeleteEntryMetaAndData(_ context.Context, p util.FullPath, _, _, _, _ bool, _ []int32, _ time.Time) error {
if m.deleteFn == nil {
return nil
}
return m.deleteFn(p)
}
func (m *mockFilerOps) GetEntryAttributes(_ context.Context, p util.FullPath) (map[string][]byte, error) {
if m.attrsFn == nil {
return nil, nil
}
return m.attrsFn(p)
}
func (m *mockFilerOps) IsDirectoryKeyObject(_ context.Context, p util.FullPath) (bool, error) {
if m.isDirKeyObjFn == nil {
return false, nil
}
return m.isDirKeyObjFn(p)
}
func Test_isUnderPath(t *testing.T) {
tests := []struct {
name string
child string
parent string
expected bool
}{
{"child under parent", "/buckets/mybucket/folder/file.txt", "/buckets", true},
{"child is parent", "/buckets", "/buckets", true},
{"child not under parent", "/other/path", "/buckets", false},
{"empty parent", "/any/path", "", true},
{"root parent", "/any/path", "/", true},
{"parent with trailing slash", "/buckets/mybucket", "/buckets/", true},
{"similar prefix but not under", "/buckets-other/file", "/buckets", false},
{"deeply nested", "/buckets/a/b/c/d/e/f", "/buckets/a/b", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := isUnderPath(tt.child, tt.parent)
if result != tt.expected {
t.Errorf("isUnderPath(%q, %q) = %v, want %v", tt.child, tt.parent, result, tt.expected)
}
})
}
}
func Test_isUnderBucketPath(t *testing.T) {
tests := []struct {
name string
directory string
bucketPath string
expected bool
}{
// Should NOT process - bucket path itself
{"bucket path itself", "/buckets", "/buckets", false},
// Should NOT process - bucket directory (immediate child)
{"bucket directory", "/buckets/mybucket", "/buckets", false},
// Should process - folder inside bucket
{"folder in bucket", "/buckets/mybucket/folder", "/buckets", true},
// Should process - nested folder
{"nested folder", "/buckets/mybucket/a/b/c", "/buckets", true},
// Should NOT process - outside buckets
{"outside buckets", "/other/path", "/buckets", false},
// Empty bucket path allows all
{"empty bucket path", "/any/path", "", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := isUnderBucketPath(tt.directory, tt.bucketPath)
if result != tt.expected {
t.Errorf("isUnderBucketPath(%q, %q) = %v, want %v", tt.directory, tt.bucketPath, result, tt.expected)
}
})
}
}
func Test_isMultipartUploadsPath(t *testing.T) {
uploads := s3_constants.MultipartUploadsFolder
tests := []struct {
name string
directory string
bucketPath string
expected bool
}{
{"staging root", "/buckets/mybucket/" + uploads, "/buckets", true},
{"upload marker dir", "/buckets/mybucket/" + uploads + "/abc123", "/buckets", true},
{"nested under staging", "/buckets/mybucket/" + uploads + "/abc123/hashstates", "/buckets", true},
{"normal folder", "/buckets/mybucket/folder", "/buckets", false},
{"registry own uploads dir", "/buckets/mybucket/docker/registry/v2/_uploads/x", "/buckets", false},
{"bucket itself", "/buckets/mybucket", "/buckets", false},
{"outside buckets", "/other/" + uploads, "/buckets", false},
{"empty bucket path", "/buckets/mybucket/" + uploads, "", false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := isMultipartUploadsPath(tt.bucketPath, tt.directory); got != tt.expected {
t.Errorf("isMultipartUploadsPath(%q, %q) = %v, want %v", tt.bucketPath, tt.directory, got, tt.expected)
}
})
}
}
func TestEmptyFolderCleaner_OnDeleteEvent_skipsMultipartUploads(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
now := time.Now()
uploads := "/buckets/mybucket/" + s3_constants.MultipartUploadsFolder
cleaner.OnDeleteEvent(uploads, "abc123", true, now)
cleaner.OnDeleteEvent(uploads+"/abc123", "0001.part", false, now)
if cleaner.GetPendingCleanupCount() != 0 {
t.Fatalf("multipart staging paths must not be queued, got %d pending", cleaner.GetPendingCleanupCount())
}
// A normal folder is still queued, proving the guard is scoped to .uploads.
cleaner.OnDeleteEvent("/buckets/mybucket/folder", "file.txt", false, now)
if cleaner.GetPendingCleanupCount() != 1 {
t.Fatalf("normal folder should be queued, got %d pending", cleaner.GetPendingCleanupCount())
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_executeCleanup_skipsMultipartUploads(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
var deleted []string
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 0, nil },
deleteFn: func(p util.FullPath) error { deleted = append(deleted, string(p)); return nil },
}
cleaner := &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
maxCountCheck: DefaultMaxCountCheck,
cacheExpiry: DefaultCacheExpiry,
folderCounts: make(map[string]*folderState),
bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
uploads := "/buckets/mybucket/" + s3_constants.MultipartUploadsFolder
cleaner.executeCleanup(uploads, "abc123")
cleaner.executeCleanup(uploads+"/abc123", "0001.part")
if len(deleted) != 0 {
t.Fatalf("multipart staging paths must not be deleted, got %v", deleted)
}
// An empty normal folder is deleted, proving the guard did not disable cleanup.
cleaner.executeCleanup("/buckets/mybucket/folder", "file.txt")
if len(deleted) != 1 || deleted[0] != "/buckets/mybucket/folder" {
t.Fatalf("normal empty folder should be deleted, got %v", deleted)
}
}
func TestEmptyFolderCleaner_restoreFoldersWrittenDuringDelete(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
newCleaner := func(mock *mockFilerOps) *EmptyFolderCleaner {
return &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
maxCountCheck: DefaultMaxCountCheck,
cacheExpiry: DefaultCacheExpiry,
folderCounts: make(map[string]*folderState),
bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
deleted: make(map[string]*deletedFolder),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
}
const folder = "/buckets/mybucket/folder"
t.Run("a create event for the deleted folder puts it back", func(t *testing.T) {
var restored []string
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 0, nil },
ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "file.txt")
// nothing has been written to it, so it is only observed
cleaner.processCleanupQueue()
if len(restored) != 0 {
t.Fatalf("a folder nothing was written to should not be restored, got %v", restored)
}
if len(cleaner.deleted) != 1 {
t.Fatalf("folder should still be observed, got %d", len(cleaner.deleted))
}
// the write that raced the delete arrives as a create event
mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
cleaner.OnCreateEvent(folder, "obj", false)
cleaner.processCleanupQueue()
if len(restored) != 1 || restored[0] != folder {
t.Fatalf("a folder written to while being deleted should be restored, got %v", restored)
}
if len(cleaner.deleted) != 0 {
t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
}
})
t.Run("a create event elsewhere is ignored", func(t *testing.T) {
var restored []string
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 0, nil },
ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "file.txt")
cleaner.OnCreateEvent("/buckets/mybucket/other", "obj", false)
cleaner.processCleanupQueue()
if len(restored) != 0 {
t.Fatalf("only the folder written to should be restored, got %v", restored)
}
})
t.Run("an entry that went away again is not restored", func(t *testing.T) {
var restored []string
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 0, nil },
ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "file.txt")
cleaner.OnCreateEvent(folder, "obj", false)
cleaner.processCleanupQueue()
if len(restored) != 0 {
t.Fatalf("an empty folder should not be restored, got %v", restored)
}
})
t.Run("observation expires", func(t *testing.T) {
mock := &mockFilerOps{countFn: func(util.FullPath) (int, error) { return 0, nil }}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "file.txt")
cleaner.processCleanupQueue()
if len(cleaner.deleted) != 1 {
t.Fatalf("folder should still be observed inside the window, got %d", len(cleaner.deleted))
}
cleaner.deleted[folder].deletedAt = time.Now().Add(-DefaultObservationWindow - time.Second)
cleaner.processCleanupQueue()
if len(cleaner.deleted) != 0 {
t.Fatalf("folder should stop being observed once the window has passed, got %d", len(cleaner.deleted))
}
})
t.Run("a failed restore is retried", func(t *testing.T) {
attempts := 0
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 1, nil },
ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error {
attempts++
if attempts == 1 {
return errors.New("store unavailable")
}
return nil
},
}
cleaner := newCleaner(mock)
// deleted while empty, then written to
mock.countFn = func(util.FullPath) (int, error) { return 0, nil }
cleaner.executeCleanup(folder, "file.txt")
mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
cleaner.OnCreateEvent(folder, "obj", false)
cleaner.processCleanupQueue()
if len(cleaner.deleted) != 1 {
t.Fatalf("a folder whose restore failed should be kept, got %d", len(cleaner.deleted))
}
cleaner.processCleanupQueue()
if attempts != 2 {
t.Fatalf("a failed restore should be retried, got %d attempts", attempts)
}
if len(cleaner.deleted) != 0 {
t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
}
})
t.Run("a folder whose restore keeps failing stops being observed", func(t *testing.T) {
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 0, nil },
ensureDirFn: func(util.FullPath, DirectoryAttributes) error { return errors.New("store unavailable") },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "file.txt")
mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
cleaner.OnCreateEvent(folder, "obj", false)
cleaner.processCleanupQueue()
if len(cleaner.deleted) != 1 {
t.Fatalf("a folder whose restore failed should be kept, got %d", len(cleaner.deleted))
}
// otherwise it would be retried on every pass for the rest of the process
cleaner.deleted[folder].deletedAt = time.Now().Add(-DefaultObservationWindow - time.Second)
cleaner.processCleanupQueue()
if len(cleaner.deleted) != 0 {
t.Fatalf("the window should apply to a failing folder too, got %d", len(cleaner.deleted))
}
})
t.Run("an ancestor taken by the cascade is restored with its own attributes", func(t *testing.T) {
const ancestor = "/buckets/mybucket/data"
const nested = ancestor + "/abc"
restored := map[string]DirectoryAttributes{}
var order []string
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 0, nil },
dirAttrsFn: func(p util.FullPath) (DirectoryAttributes, error) {
if string(p) == ancestor {
return DirectoryAttributes{Mode: 0700, Uid: 4242}, nil
}
return DirectoryAttributes{Mode: 0755, Uid: 7}, nil
},
ensureDirFn: func(p util.FullPath, attrs DirectoryAttributes) error {
restored[string(p)] = attrs
order = append(order, string(p))
return nil
},
}
cleaner := newCleaner(mock)
// the cascade takes the nested folder and then its parent
cleaner.executeCleanup(nested, "file.txt")
cleaner.executeCleanup(ancestor, "abc")
// only the nested folder is written to
mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
cleaner.OnCreateEvent(nested, "obj", false)
cleaner.processCleanupQueue()
if len(order) != 2 || order[0] != ancestor {
t.Fatalf("the ancestor should be rebuilt first, got %v", order)
}
if got := restored[ancestor]; got.Mode != 0700 || got.Uid != 4242 {
t.Errorf("ancestor should keep its own attributes, got mode %o uid %d", got.Mode, got.Uid)
}
})
t.Run("a delete that reports failure is still observed", func(t *testing.T) {
// the redis stores drop the folder before its parent-list member, so a failure
// return is not proof that the folder survived
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 0, nil },
deleteFn: func(util.FullPath) error { return errors.New("partial delete") },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "file.txt")
if len(cleaner.deleted) != 1 {
t.Fatalf("a failed delete should still leave the folder observed, got %d", len(cleaner.deleted))
}
})
}
func Test_autoRemoveEmptyFoldersEnabled(t *testing.T) {
tests := []struct {
name string
attrs map[string][]byte
enabled bool
attrValue string
}{
{
name: "no attrs defaults enabled",
attrs: nil,
enabled: true,
attrValue: "<no_attrs>",
},
{
name: "missing key defaults enabled",
attrs: map[string][]byte{},
enabled: true,
attrValue: "<missing>",
},
{
name: "allow-empty disables cleanup",
attrs: map[string][]byte{
s3_constants.ExtAllowEmptyFolders: []byte("true"),
},
enabled: false,
attrValue: "true",
},
{
name: "explicit false keeps cleanup enabled",
attrs: map[string][]byte{
s3_constants.ExtAllowEmptyFolders: []byte("false"),
},
enabled: true,
attrValue: "false",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
enabled, attrValue := autoRemoveEmptyFoldersEnabled(tt.attrs)
if enabled != tt.enabled {
t.Fatalf("expected enabled=%v, got %v", tt.enabled, enabled)
}
if attrValue != tt.attrValue {
t.Fatalf("expected attrValue=%q, got %q", tt.attrValue, attrValue)
}
})
}
}
func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
// Create a LockRing with multiple servers
lockRing := lock_manager.NewLockRing(5 * time.Second)
servers := []pb.ServerAddress{
"filer1:8888",
"filer2:8888",
"filer3:8888",
}
lockRing.SetSnapshot(servers, 0)
// Create cleaner for filer1
cleaner1 := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
}
// Create cleaner for filer2
cleaner2 := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer2:8888",
}
// Create cleaner for filer3
cleaner3 := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer3:8888",
}
// Test that exactly one filer owns each folder
testFolders := []string{
"/buckets/mybucket/folder1",
"/buckets/mybucket/folder2",
"/buckets/mybucket/folder3",
"/buckets/mybucket/a/b/c",
"/buckets/otherbucket/x",
}
for _, folder := range testFolders {
ownCount := 0
if cleaner1.ownsFolder(folder) {
ownCount++
}
if cleaner2.ownsFolder(folder) {
ownCount++
}
if cleaner3.ownsFolder(folder) {
ownCount++
}
if ownCount != 1 {
t.Errorf("folder %q owned by %d filers, expected exactly 1", folder, ownCount)
}
}
}
func TestEmptyFolderCleaner_ownsFolder_singleServer(t *testing.T) {
// Create a LockRing with a single server
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
}
// Single filer should own all folders
testFolders := []string{
"/buckets/mybucket/folder1",
"/buckets/mybucket/folder2",
"/buckets/otherbucket/x",
}
for _, folder := range testFolders {
if !cleaner.ownsFolder(folder) {
t.Errorf("single filer should own folder %q", folder)
}
}
}
func TestEmptyFolderCleaner_ownsFolder_emptyRing(t *testing.T) {
// Create an empty LockRing
lockRing := lock_manager.NewLockRing(5 * time.Second)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
}
// With empty ring, should own all folders
if !cleaner.ownsFolder("/buckets/mybucket/folder") {
t.Error("should own folder with empty ring")
}
}
func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
folder := "/buckets/mybucket/testfolder"
now := time.Now()
// Simulate delete event
cleaner.OnDeleteEvent(folder, "file.txt", false, now)
// Check that cleanup is queued
if cleaner.GetPendingCleanupCount() != 1 {
t.Errorf("expected 1 pending cleanup, got %d", cleaner.GetPendingCleanupCount())
}
// Simulate create event
cleaner.OnCreateEvent(folder, "newfile.txt", false)
// Check that cleanup is cancelled
if cleaner.GetPendingCleanupCount() != 0 {
t.Errorf("expected 0 pending cleanups after create, got %d", cleaner.GetPendingCleanupCount())
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
folder := "/buckets/mybucket/testfolder"
now := time.Now()
// Simulate multiple delete events for same folder
for i := 0; i < 5; i++ {
cleaner.OnDeleteEvent(folder, "file"+string(rune('0'+i))+".txt", false, now.Add(time.Duration(i)*time.Second))
}
// Check that only 1 cleanup is queued (deduplicated)
if cleaner.GetPendingCleanupCount() != 1 {
t.Errorf("expected 1 pending cleanup after deduplication, got %d", cleaner.GetPendingCleanupCount())
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
now := time.Now()
// Delete files in different folders
cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file.txt", false, now)
cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file.txt", false, now.Add(1*time.Second))
cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file.txt", false, now.Add(2*time.Second))
// Each folder should be queued
if cleaner.GetPendingCleanupCount() != 3 {
t.Errorf("expected 3 pending cleanups, got %d", cleaner.GetPendingCleanupCount())
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"}, 0)
// Create cleaner for filer that doesn't own the folder
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
now := time.Now()
// Try many folders, looking for one that filer1 doesn't own
foundNonOwned := false
for i := 0; i < 100; i++ {
folder := "/buckets/mybucket/folder" + string(rune('0'+i%10)) + string(rune('0'+i/10))
if !cleaner.ownsFolder(folder) {
// This folder is not owned by filer1
cleaner.OnDeleteEvent(folder, "file.txt", false, now)
if cleaner.GetPendingCleanupCount() != 0 {
t.Errorf("non-owner should not queue cleanup for folder %s", folder)
}
foundNonOwned = true
break
}
}
if !foundNonOwned {
t.Skip("could not find a folder not owned by filer1")
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: false, // Disabled
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
folder := "/buckets/mybucket/testfolder"
now := time.Now()
// Simulate delete event
cleaner.OnDeleteEvent(folder, "file.txt", false, now)
// Check that no cleanup is queued when disabled
if cleaner.GetPendingCleanupCount() != 0 {
t.Errorf("disabled cleaner should not queue cleanup, got %d", cleaner.GetPendingCleanupCount())
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
folder := "/buckets/mybucket/testfolder"
now := time.Now()
// Simulate directory delete event - should trigger cleanup
// because subdirectory deletion also makes parent potentially empty
cleaner.OnDeleteEvent(folder, "subdir", true, now)
// Check that cleanup IS queued for directory deletion
if cleaner.GetPendingCleanupCount() != 1 {
t.Errorf("directory deletion should trigger cleanup, got %d", cleaner.GetPendingCleanupCount())
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
folder := "/buckets/mybucket/testfolder"
// Initialize cached count
cleaner.folderCounts[folder] = &folderState{roughCount: 5}
// Simulate create events
cleaner.OnCreateEvent(folder, "newfile1.txt", false)
cleaner.OnCreateEvent(folder, "newfile2.txt", false)
// Check cached count increased
count, exists := cleaner.GetCachedFolderCount(folder)
if !exists {
t.Error("cached folder count should exist")
}
if count != 7 {
t.Errorf("expected cached count 7, got %d", count)
}
// Simulate delete events
now := time.Now()
cleaner.OnDeleteEvent(folder, "file1.txt", false, now)
cleaner.OnDeleteEvent(folder, "file2.txt", false, now.Add(1*time.Second))
// Check cached count decreased
count, exists = cleaner.GetCachedFolderCount(folder)
if !exists {
t.Error("cached folder count should exist")
}
if count != 5 {
t.Errorf("expected cached count 5, got %d", count)
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_Stop(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
now := time.Now()
// Queue some cleanups
cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file1.txt", false, now)
cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file2.txt", false, now.Add(1*time.Second))
cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file3.txt", false, now.Add(2*time.Second))
// Verify cleanups are queued
if cleaner.GetPendingCleanupCount() < 1 {
t.Error("expected at least 1 pending cleanup before stop")
}
// Stop the cleaner
cleaner.Stop()
// Verify all cleanups are cancelled
if cleaner.GetPendingCleanupCount() != 0 {
t.Errorf("expected 0 pending cleanups after stop, got %d", cleaner.GetPendingCleanupCount())
}
}
func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
cacheExpiry: 100 * time.Millisecond, // Short expiry for testing
stopCh: make(chan struct{}),
}
folder1 := "/buckets/mybucket/folder1"
folder2 := "/buckets/mybucket/folder2"
folder3 := "/buckets/mybucket/folder3"
// Add some cache entries with old timestamps
oldTime := time.Now().Add(-1 * time.Hour)
cleaner.folderCounts[folder1] = &folderState{roughCount: 5, lastCheck: oldTime}
cleaner.folderCounts[folder2] = &folderState{roughCount: 3, lastCheck: oldTime}
// folder3 has recent activity
cleaner.folderCounts[folder3] = &folderState{roughCount: 2, lastCheck: time.Now()}
// Verify all entries exist
if len(cleaner.folderCounts) != 3 {
t.Errorf("expected 3 cache entries, got %d", len(cleaner.folderCounts))
}
// Run eviction
cleaner.evictStaleCacheEntries()
// Verify stale entries are evicted
if len(cleaner.folderCounts) != 1 {
t.Errorf("expected 1 cache entry after eviction, got %d", len(cleaner.folderCounts))
}
// Verify the recent entry still exists
if _, exists := cleaner.folderCounts[folder3]; !exists {
t.Error("expected folder3 to still exist in cache")
}
// Verify stale entries are removed
if _, exists := cleaner.folderCounts[folder1]; exists {
t.Error("expected folder1 to be evicted")
}
if _, exists := cleaner.folderCounts[folder2]; exists {
t.Error("expected folder2 to be evicted")
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
cacheExpiry: 100 * time.Millisecond,
stopCh: make(chan struct{}),
}
folder := "/buckets/mybucket/folder"
oldTime := time.Now().Add(-1 * time.Hour)
// Add a stale cache entry
cleaner.folderCounts[folder] = &folderState{roughCount: 0, lastCheck: oldTime}
// Also add to cleanup queue
cleaner.cleanupQueue.Add(folder, "item", time.Now())
// Run eviction
cleaner.evictStaleCacheEntries()
// Verify entry is NOT evicted because it's in cleanup queue
if _, exists := cleaner.folderCounts[folder]; !exists {
t.Error("expected folder to still exist in cache (is in cleanup queue)")
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
now := time.Now()
// Add folders in order
folders := []string{
"/buckets/mybucket/folder1",
"/buckets/mybucket/folder2",
"/buckets/mybucket/folder3",
}
for i, folder := range folders {
cleaner.OnDeleteEvent(folder, "file.txt", false, now.Add(time.Duration(i)*time.Second))
}
// Verify queue length
if cleaner.GetPendingCleanupCount() != 3 {
t.Errorf("expected 3 queued folders, got %d", cleaner.GetPendingCleanupCount())
}
// Verify time-sorted order by popping
for i, expected := range folders {
folder, _, ok := cleaner.cleanupQueue.Pop()
if !ok || folder != expected {
t.Errorf("expected folder %s at index %d, got %s", expected, i, folder)
}
}
cleaner.Stop()
}
func TestEmptyFolderCleaner_processCleanupQueue_onlyProcessesAgedItems(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
var deleted []string
mock := &mockFilerOps{
countFn: func(_ util.FullPath) (int, error) {
return 0, nil
},
deleteFn: func(path util.FullPath) error {
deleted = append(deleted, string(path))
return nil
},
}
maxAge := 100 * time.Millisecond
cleaner := &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, maxAge),
maxCountCheck: 1000,
cacheExpiry: time.Minute,
processorSleep: time.Second,
stopCh: make(chan struct{}),
}
// Add old items (well past maxAge) and a fresh item
old := time.Now().Add(-time.Second)
cleaner.cleanupQueue.Add("/buckets/test/folder1", "i1", old)
cleaner.cleanupQueue.Add("/buckets/test/folder2", "i2", old.Add(time.Millisecond))
cleaner.cleanupQueue.Add("/buckets/test/folder3", "i3", time.Now()) // fresh, should NOT be processed
cleaner.processCleanupQueue()
// Only the two old items should have been processed
if len(deleted) != 2 {
t.Fatalf("expected 2 deleted folders (aged items only), got %d: %v", len(deleted), deleted)
}
if got := cleaner.cleanupQueue.Len(); got != 1 {
t.Fatalf("expected 1 item remaining in queue, got %d", got)
}
}
func TestEmptyFolderCleaner_executeCleanup_bucketPolicyDisabledSkips(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
var deleted []string
mock := &mockFilerOps{
countFn: func(_ util.FullPath) (int, error) {
return 0, nil
},
deleteFn: func(path util.FullPath) error {
deleted = append(deleted, string(path))
return nil
},
attrsFn: func(path util.FullPath) (map[string][]byte, error) {
if string(path) == "/buckets/test" {
return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("true")}, nil
}
return nil, nil
},
}
cleaner := &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, time.Minute),
maxCountCheck: 1000,
cacheExpiry: time.Minute,
processorSleep: time.Second,
stopCh: make(chan struct{}),
}
folder := "/buckets/test/folder"
cleaner.executeCleanup(folder, "triggered_item")
if len(deleted) != 0 {
t.Fatalf("expected folder %s to be skipped, got deletions %v", folder, deleted)
}
}
func TestEmptyFolderCleaner_InvalidateBucketPolicy(t *testing.T) {
cleaner := &EmptyFolderCleaner{
bucketPath: "/buckets",
enabled: true,
bucketCleanupPolicies: map[string]*bucketCleanupPolicyState{
"/buckets/test": {},
},
}
cleaner.InvalidateBucketPolicy("/buckets", "test", true)
if _, found := cleaner.bucketCleanupPolicies["/buckets/test"]; found {
t.Fatal("expected cached bucket policy to be evicted")
}
cleaner.bucketCleanupPolicies["/buckets/test"] = &bucketCleanupPolicyState{}
cleaner.InvalidateBucketPolicy("/buckets/test", "dir", true)
cleaner.InvalidateBucketPolicy("/buckets", "test", false)
if _, found := cleaner.bucketCleanupPolicies["/buckets/test"]; !found {
t.Fatal("expected unrelated updates to keep the cached policy")
}
}
func TestEmptyFolderCleaner_getBucketCleanupPolicy_concurrentUpdate(t *testing.T) {
var cleaner *EmptyFolderCleaner
calls := 0
mock := &mockFilerOps{
attrsFn: func(_ util.FullPath) (map[string][]byte, error) {
calls++
if calls == 1 {
// a toggle lands while the stale read is in flight
cleaner.InvalidateBucketPolicy("/buckets", "test", true)
return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("true")}, nil
}
return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("false")}, nil
},
}
cleaner = &EmptyFolderCleaner{
filer: mock,
bucketPath: "/buckets",
enabled: true,
bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
}
_, autoRemove, _, attrValue, err := cleaner.getBucketCleanupPolicy(context.Background(), "/buckets/test/folder")
if err != nil {
t.Fatalf("getBucketCleanupPolicy: %v", err)
}
if !autoRemove || attrValue != "false" {
t.Fatalf("expected the post-update policy autoRemove=true attr=false, got autoRemove=%v attr=%q", autoRemove, attrValue)
}
if calls < 2 {
t.Fatalf("expected the stale load to be retried, got %d attribute reads", calls)
}
if cached := cleaner.bucketCleanupPolicies["/buckets/test"]; cached == nil || cached.attrValue != "false" {
t.Fatalf("expected the fresh policy to be cached, got %+v", cached)
}
}
func TestEmptyFolderCleaner_getBucketCleanupPolicy_evictionKeepsInflightRead(t *testing.T) {
var cleaner *EmptyFolderCleaner
started := make(chan struct{})
release := make(chan struct{})
calls := 0
mock := &mockFilerOps{
attrsFn: func(_ util.FullPath) (map[string][]byte, error) {
calls++
if calls == 1 {
close(started)
<-release
return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("true")}, nil
}
return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("false")}, nil
},
}
cleaner = &EmptyFolderCleaner{
filer: mock,
bucketPath: "/buckets",
enabled: true,
cacheExpiry: time.Minute,
folderCounts: make(map[string]*folderState),
bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
policyGen: make(map[string]*bucketPolicyGen),
cleanupQueue: NewCleanupQueue(1000, time.Minute),
}
go func() {
<-started
cleaner.InvalidateBucketPolicy("/buckets", "test", true)
cleaner.mu.Lock()
cleaner.policyGen["/buckets/test"].lastBump = time.Now().Add(-time.Hour)
cleaner.mu.Unlock()
cleaner.evictStaleCacheEntries()
close(release)
}()
_, autoRemove, _, attrValue, err := cleaner.getBucketCleanupPolicy(context.Background(), "/buckets/test/folder")
if err != nil {
t.Fatalf("getBucketCleanupPolicy: %v", err)
}
if !autoRemove || attrValue != "false" {
t.Fatalf("expected the stalled read to be discarded, got autoRemove=%v attr=%q", autoRemove, attrValue)
}
if calls < 2 {
t.Fatalf("expected the stale load to be retried, got %d attribute reads", calls)
}
}
func TestEmptyFolderCleaner_executeCleanup_policyFailureRequeues(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
mock := &mockFilerOps{
attrsFn: func(_ util.FullPath) (map[string][]byte, error) {
return nil, errors.New("attrs unavailable")
},
}
cleaner := &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
cleanupQueue: NewCleanupQueue(1000, time.Millisecond),
stopCh: make(chan struct{}),
}
folder := "/buckets/test/folder"
for i := 0; i < DefaultMaxPolicyFailures; i++ {
cleaner.executeCleanup(folder, "triggered_item")
popped, _, ok := cleaner.cleanupQueue.Pop()
if !ok || popped != folder {
t.Fatalf("failure %d of %d should requeue the folder", i+1, DefaultMaxPolicyFailures)
}
}
cleaner.executeCleanup(folder, "triggered_item")
if got := cleaner.folderCounts[folder].policyFailures; got != DefaultMaxPolicyFailures+1 {
t.Fatalf("policyFailures = %d, want %d", got, DefaultMaxPolicyFailures+1)
}
if got := cleaner.cleanupQueue.Len(); got != 0 {
t.Fatalf("expected retries to stop after %d failures, got %d queued", DefaultMaxPolicyFailures, got)
}
}
func TestEmptyFolderCleaner_executeCleanup_directoryMarker(t *testing.T) {
testCases := []struct {
name string
isDirKeyObj bool
expectDeletion bool
}{
{
name: "skips explicit directory marker",
isDirKeyObj: true,
expectDeletion: false,
},
{
name: "deletes implicit empty folder",
isDirKeyObj: false,
expectDeletion: true,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
var deleted []string
mock := &mockFilerOps{
countFn: func(_ util.FullPath) (int, error) {
return 0, nil
},
deleteFn: func(path util.FullPath) error {
deleted = append(deleted, string(path))
return nil
},
isDirKeyObjFn: func(path util.FullPath) (bool, error) {
return tc.isDirKeyObj, nil
},
}
cleaner := &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, time.Minute),
maxCountCheck: 1000,
cacheExpiry: time.Minute,
processorSleep: time.Second,
stopCh: make(chan struct{}),
}
folder := "/buckets/test/folder"
cleaner.executeCleanup(folder, "triggered_item")
if tc.expectDeletion {
if len(deleted) != 1 || deleted[0] != folder {
t.Fatalf("expected implicit empty folder %s to be deleted, got deletions %v", folder, deleted)
}
} else {
if len(deleted) != 0 {
t.Fatalf("expected explicit directory marker %s to be preserved, got deletions %v", folder, deleted)
}
}
})
}
}
func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanupForRecreatedDirectory(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
cleaner := &EmptyFolderCleaner{
filer: &mockFilerOps{},
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
deleted: make(map[string]*deletedFolder),
cleanupQueue: NewCleanupQueue(1000, time.Minute),
maxCountCheck: 1000,
cacheExpiry: time.Minute,
processorSleep: time.Second,
stopCh: make(chan struct{}),
}
// Dropping the table queues the folder its children were removed from, then a
// rename puts a new table back under the same name.
cleaner.OnDeleteEvent("/buckets/warehouse/pedsnet/person", "metadata", true, time.Now())
if cleaner.GetPendingCleanupCount() != 1 {
t.Fatalf("expected the emptied folder to be queued, got %d", cleaner.GetPendingCleanupCount())
}
cleaner.OnCreateEvent("/buckets/warehouse/pedsnet", "person", true)
if cleaner.GetPendingCleanupCount() != 0 {
t.Fatalf("expected the recreated folder to leave the queue, got %d", cleaner.GetPendingCleanupCount())
}
}
func TestEmptyFolderCleaner_executeCleanup_skipsCatalogEntry(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
const namespace = "/buckets/warehouse/pedsnet"
const table = namespace + "/person"
var deleted []string
mock := &mockFilerOps{
countFn: func(_ util.FullPath) (int, error) {
return 0, nil
},
deleteFn: func(path util.FullPath) error {
deleted = append(deleted, string(path))
return nil
},
attrsFn: func(path util.FullPath) (map[string][]byte, error) {
if path == namespace || path == table {
return map[string][]byte{s3_constants.ExtS3TablesPrefix + "metadata": []byte("{}")}, nil
}
return nil, nil
},
}
cleaner := &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
folderCounts: make(map[string]*folderState),
cleanupQueue: NewCleanupQueue(1000, time.Minute),
maxCountCheck: 1000,
cacheExpiry: time.Minute,
processorSleep: time.Second,
stopCh: make(chan struct{}),
}
// The dropped table left its metadata folder empty; the cascade up from it must
// stop at the table the rename put back, and at the namespace above it.
cleaner.executeCleanup(table+"/metadata", "v2.metadata.json")
if len(deleted) != 1 || deleted[0] != table+"/metadata" {
t.Fatalf("expected only %s to be deleted, got %v", table+"/metadata", deleted)
}
}
func TestEmptyFolderCleaner_restoreIfWrittenTo(t *testing.T) {
lockRing := lock_manager.NewLockRing(5 * time.Second)
lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)
newCleaner := func(mock *mockFilerOps) *EmptyFolderCleaner {
return &EmptyFolderCleaner{
filer: mock,
lockRing: lockRing,
host: "filer1:8888",
bucketPath: "/buckets",
enabled: true,
maxCountCheck: DefaultMaxCountCheck,
cacheExpiry: DefaultCacheExpiry,
folderCounts: make(map[string]*folderState),
bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
deleted: make(map[string]*deletedFolder),
cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
stopCh: make(chan struct{}),
}
}
const folder = "/buckets/mybucket/a/b"
const parent = "/buckets/mybucket/a"
t.Run("an entry that landed during the delete is restored without waiting for its event", func(t *testing.T) {
counts := map[string]int{}
var deleted, restored []string
mock := &mockFilerOps{
countFn: func(p util.FullPath) (int, error) { return counts[string(p)], nil },
deleteFn: func(p util.FullPath) error {
deleted = append(deleted, string(p))
if string(p) == folder {
// the racing write lands between the delete's own emptiness
// check and the removal of the folder entry
counts[folder] = 1
}
return nil
},
ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "obj")
if len(restored) != 1 || restored[0] != folder {
t.Fatalf("folder written to during the delete should be restored, got %v", restored)
}
if len(deleted) != 1 || deleted[0] != folder {
t.Fatalf("a restored folder leaves the parent non-empty, so it must not cascade, got %v", deleted)
}
if len(cleaner.deleted) != 0 {
t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
}
})
t.Run("a folder still empty after the delete is not restored and the parent is cascaded", func(t *testing.T) {
var deleted, restored []string
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) { return 0, nil },
deleteFn: func(p util.FullPath) error { deleted = append(deleted, string(p)); return nil },
ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "obj")
if len(restored) != 0 {
t.Fatalf("an empty folder should not be restored, got %v", restored)
}
if len(deleted) != 2 || deleted[0] != folder || deleted[1] != parent {
t.Fatalf("the parent should still be cascaded, got %v", deleted)
}
})
t.Run("a restore that fails is left marked for the deferred pass", func(t *testing.T) {
counts := map[string]int{}
mock := &mockFilerOps{
countFn: func(p util.FullPath) (int, error) { return counts[string(p)], nil },
deleteFn: func(p util.FullPath) error {
if string(p) == folder {
counts[folder] = 1
}
return nil
},
ensureDirFn: func(util.FullPath, DirectoryAttributes) error { return errors.New("store unavailable") },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "obj")
observed, found := cleaner.deleted[folder]
if !found {
t.Fatal("a folder whose restore failed must stay under observation")
}
if !observed.writtenTo {
t.Fatal("a folder seen to hold an entry must be marked, or the deferred pass skips it")
}
})
t.Run("a re-check that fails leaves the folder to the event path", func(t *testing.T) {
var restored []string
deletedFolder := false
mock := &mockFilerOps{
countFn: func(util.FullPath) (int, error) {
if deletedFolder {
return 0, errors.New("store unavailable")
}
return 0, nil
},
deleteFn: func(p util.FullPath) error { deletedFolder = string(p) == folder; return nil },
ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
}
cleaner := newCleaner(mock)
cleaner.executeCleanup(folder, "obj")
if len(restored) != 0 {
t.Fatalf("a failed re-check should not restore, got %v", restored)
}
if _, found := cleaner.deleted[folder]; !found {
t.Fatal("a folder whose re-check failed must stay under observation")
}
})
}