Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

47.4 KB
Raw Blame
package empty_folder_cleanup

import (
	"context"
	"errors"
	"testing"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

type mockFilerOps struct {
	countFn       func(path util.FullPath) (int, error)
	deleteFn      func(path util.FullPath) error
	attrsFn       func(path util.FullPath) (map[string][]byte, error)
	isDirKeyObjFn func(path util.FullPath) (bool, error)
	ensureDirFn   func(path util.FullPath, attrs DirectoryAttributes) error
	dirAttrsFn    func(path util.FullPath) (DirectoryAttributes, error)
}

func (m *mockFilerOps) EnsureDirectoryEntry(_ context.Context, p util.FullPath, attrs DirectoryAttributes) error {
	if m.ensureDirFn == nil {
		return nil
	}
	return m.ensureDirFn(p, attrs)
}

func (m *mockFilerOps) DirectoryAttributes(_ context.Context, p util.FullPath) (DirectoryAttributes, error) {
	if m.dirAttrsFn == nil {
		return DirectoryAttributes{Mode: 0755}, nil
	}
	return m.dirAttrsFn(p)
}

func (m *mockFilerOps) CountDirectoryEntries(_ context.Context, dirPath util.FullPath, _ int) (int, error) {
	if m.countFn == nil {
		return 0, nil
	}
	return m.countFn(dirPath)
}

func (m *mockFilerOps) DeleteEntryMetaAndData(_ context.Context, p util.FullPath, _, _, _, _ bool, _ []int32, _ time.Time) error {
	if m.deleteFn == nil {
		return nil
	}
	return m.deleteFn(p)
}

func (m *mockFilerOps) GetEntryAttributes(_ context.Context, p util.FullPath) (map[string][]byte, error) {
	if m.attrsFn == nil {
		return nil, nil
	}
	return m.attrsFn(p)
}

func (m *mockFilerOps) IsDirectoryKeyObject(_ context.Context, p util.FullPath) (bool, error) {
	if m.isDirKeyObjFn == nil {
		return false, nil
	}
	return m.isDirKeyObjFn(p)
}

func Test_isUnderPath(t *testing.T) {
	tests := []struct {
		name     string
		child    string
		parent   string
		expected bool
	}{
		{"child under parent", "/buckets/mybucket/folder/file.txt", "/buckets", true},
		{"child is parent", "/buckets", "/buckets", true},
		{"child not under parent", "/other/path", "/buckets", false},
		{"empty parent", "/any/path", "", true},
		{"root parent", "/any/path", "/", true},
		{"parent with trailing slash", "/buckets/mybucket", "/buckets/", true},
		{"similar prefix but not under", "/buckets-other/file", "/buckets", false},
		{"deeply nested", "/buckets/a/b/c/d/e/f", "/buckets/a/b", true},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := isUnderPath(tt.child, tt.parent)
			if result != tt.expected {
				t.Errorf("isUnderPath(%q, %q) = %v, want %v", tt.child, tt.parent, result, tt.expected)
			}
		})
	}
}

func Test_isUnderBucketPath(t *testing.T) {
	tests := []struct {
		name       string
		directory  string
		bucketPath string
		expected   bool
	}{
		// Should NOT process - bucket path itself
		{"bucket path itself", "/buckets", "/buckets", false},
		// Should NOT process - bucket directory (immediate child)
		{"bucket directory", "/buckets/mybucket", "/buckets", false},
		// Should process - folder inside bucket
		{"folder in bucket", "/buckets/mybucket/folder", "/buckets", true},
		// Should process - nested folder
		{"nested folder", "/buckets/mybucket/a/b/c", "/buckets", true},
		// Should NOT process - outside buckets
		{"outside buckets", "/other/path", "/buckets", false},
		// Empty bucket path allows all
		{"empty bucket path", "/any/path", "", true},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			result := isUnderBucketPath(tt.directory, tt.bucketPath)
			if result != tt.expected {
				t.Errorf("isUnderBucketPath(%q, %q) = %v, want %v", tt.directory, tt.bucketPath, result, tt.expected)
			}
		})
	}
}

func Test_isMultipartUploadsPath(t *testing.T) {
	uploads := s3_constants.MultipartUploadsFolder
	tests := []struct {
		name       string
		directory  string
		bucketPath string
		expected   bool
	}{
		{"staging root", "/buckets/mybucket/" + uploads, "/buckets", true},
		{"upload marker dir", "/buckets/mybucket/" + uploads + "/abc123", "/buckets", true},
		{"nested under staging", "/buckets/mybucket/" + uploads + "/abc123/hashstates", "/buckets", true},
		{"normal folder", "/buckets/mybucket/folder", "/buckets", false},
		{"registry own uploads dir", "/buckets/mybucket/docker/registry/v2/_uploads/x", "/buckets", false},
		{"bucket itself", "/buckets/mybucket", "/buckets", false},
		{"outside buckets", "/other/" + uploads, "/buckets", false},
		{"empty bucket path", "/buckets/mybucket/" + uploads, "", false},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := isMultipartUploadsPath(tt.bucketPath, tt.directory); got != tt.expected {
				t.Errorf("isMultipartUploadsPath(%q, %q) = %v, want %v", tt.bucketPath, tt.directory, got, tt.expected)
			}
		})
	}
}

func TestEmptyFolderCleaner_OnDeleteEvent_skipsMultipartUploads(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()
	uploads := "/buckets/mybucket/" + s3_constants.MultipartUploadsFolder
	cleaner.OnDeleteEvent(uploads, "abc123", true, now)
	cleaner.OnDeleteEvent(uploads+"/abc123", "0001.part", false, now)
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Fatalf("multipart staging paths must not be queued, got %d pending", cleaner.GetPendingCleanupCount())
	}

	// A normal folder is still queued, proving the guard is scoped to .uploads.
	cleaner.OnDeleteEvent("/buckets/mybucket/folder", "file.txt", false, now)
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Fatalf("normal folder should be queued, got %d pending", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_executeCleanup_skipsMultipartUploads(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	var deleted []string
	mock := &mockFilerOps{
		countFn:  func(util.FullPath) (int, error) { return 0, nil },
		deleteFn: func(p util.FullPath) error { deleted = append(deleted, string(p)); return nil },
	}

	cleaner := &EmptyFolderCleaner{
		filer:                 mock,
		lockRing:              lockRing,
		host:                  "filer1:8888",
		bucketPath:            "/buckets",
		enabled:               true,
		maxCountCheck:         DefaultMaxCountCheck,
		cacheExpiry:           DefaultCacheExpiry,
		folderCounts:          make(map[string]*folderState),
		bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
		cleanupQueue:          NewCleanupQueue(1000, 10*time.Minute),
		stopCh:                make(chan struct{}),
	}

	uploads := "/buckets/mybucket/" + s3_constants.MultipartUploadsFolder
	cleaner.executeCleanup(uploads, "abc123")
	cleaner.executeCleanup(uploads+"/abc123", "0001.part")
	if len(deleted) != 0 {
		t.Fatalf("multipart staging paths must not be deleted, got %v", deleted)
	}

	// An empty normal folder is deleted, proving the guard did not disable cleanup.
	cleaner.executeCleanup("/buckets/mybucket/folder", "file.txt")
	if len(deleted) != 1 || deleted[0] != "/buckets/mybucket/folder" {
		t.Fatalf("normal empty folder should be deleted, got %v", deleted)
	}
}

func TestEmptyFolderCleaner_restoreFoldersWrittenDuringDelete(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	newCleaner := func(mock *mockFilerOps) *EmptyFolderCleaner {
		return &EmptyFolderCleaner{
			filer:                 mock,
			lockRing:              lockRing,
			host:                  "filer1:8888",
			bucketPath:            "/buckets",
			enabled:               true,
			maxCountCheck:         DefaultMaxCountCheck,
			cacheExpiry:           DefaultCacheExpiry,
			folderCounts:          make(map[string]*folderState),
			bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
			deleted:               make(map[string]*deletedFolder),
			cleanupQueue:          NewCleanupQueue(1000, 10*time.Minute),
			stopCh:                make(chan struct{}),
		}
	}

	const folder = "/buckets/mybucket/folder"

	t.Run("a create event for the deleted folder puts it back", func(t *testing.T) {
		var restored []string
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")

		// nothing has been written to it, so it is only observed
		cleaner.processCleanupQueue()
		if len(restored) != 0 {
			t.Fatalf("a folder nothing was written to should not be restored, got %v", restored)
		}
		if len(cleaner.deleted) != 1 {
			t.Fatalf("folder should still be observed, got %d", len(cleaner.deleted))
		}

		// the write that raced the delete arrives as a create event
		mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
		cleaner.OnCreateEvent(folder, "obj", false)
		cleaner.processCleanupQueue()
		if len(restored) != 1 || restored[0] != folder {
			t.Fatalf("a folder written to while being deleted should be restored, got %v", restored)
		}
		if len(cleaner.deleted) != 0 {
			t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
		}
	})

	t.Run("a create event elsewhere is ignored", func(t *testing.T) {
		var restored []string
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		cleaner.OnCreateEvent("/buckets/mybucket/other", "obj", false)
		cleaner.processCleanupQueue()
		if len(restored) != 0 {
			t.Fatalf("only the folder written to should be restored, got %v", restored)
		}
	})

	t.Run("an entry that went away again is not restored", func(t *testing.T) {
		var restored []string
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		cleaner.OnCreateEvent(folder, "obj", false)
		cleaner.processCleanupQueue()
		if len(restored) != 0 {
			t.Fatalf("an empty folder should not be restored, got %v", restored)
		}
	})

	t.Run("observation expires", func(t *testing.T) {
		mock := &mockFilerOps{countFn: func(util.FullPath) (int, error) { return 0, nil }}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 1 {
			t.Fatalf("folder should still be observed inside the window, got %d", len(cleaner.deleted))
		}

		cleaner.deleted[folder].deletedAt = time.Now().Add(-DefaultObservationWindow - time.Second)
		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 0 {
			t.Fatalf("folder should stop being observed once the window has passed, got %d", len(cleaner.deleted))
		}
	})

	t.Run("a failed restore is retried", func(t *testing.T) {
		attempts := 0
		mock := &mockFilerOps{
			countFn: func(util.FullPath) (int, error) { return 1, nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error {
				attempts++
				if attempts == 1 {
					return errors.New("store unavailable")
				}
				return nil
			},
		}

		cleaner := newCleaner(mock)
		// deleted while empty, then written to
		mock.countFn = func(util.FullPath) (int, error) { return 0, nil }
		cleaner.executeCleanup(folder, "file.txt")
		mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
		cleaner.OnCreateEvent(folder, "obj", false)

		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 1 {
			t.Fatalf("a folder whose restore failed should be kept, got %d", len(cleaner.deleted))
		}

		cleaner.processCleanupQueue()
		if attempts != 2 {
			t.Fatalf("a failed restore should be retried, got %d attempts", attempts)
		}
		if len(cleaner.deleted) != 0 {
			t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
		}
	})

	t.Run("a folder whose restore keeps failing stops being observed", func(t *testing.T) {
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			ensureDirFn: func(util.FullPath, DirectoryAttributes) error { return errors.New("store unavailable") },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
		cleaner.OnCreateEvent(folder, "obj", false)

		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 1 {
			t.Fatalf("a folder whose restore failed should be kept, got %d", len(cleaner.deleted))
		}

		// otherwise it would be retried on every pass for the rest of the process
		cleaner.deleted[folder].deletedAt = time.Now().Add(-DefaultObservationWindow - time.Second)
		cleaner.processCleanupQueue()
		if len(cleaner.deleted) != 0 {
			t.Fatalf("the window should apply to a failing folder too, got %d", len(cleaner.deleted))
		}
	})

	t.Run("an ancestor taken by the cascade is restored with its own attributes", func(t *testing.T) {
		const ancestor = "/buckets/mybucket/data"
		const nested = ancestor + "/abc"

		restored := map[string]DirectoryAttributes{}
		var order []string
		mock := &mockFilerOps{
			countFn: func(util.FullPath) (int, error) { return 0, nil },
			dirAttrsFn: func(p util.FullPath) (DirectoryAttributes, error) {
				if string(p) == ancestor {
					return DirectoryAttributes{Mode: 0700, Uid: 4242}, nil
				}
				return DirectoryAttributes{Mode: 0755, Uid: 7}, nil
			},
			ensureDirFn: func(p util.FullPath, attrs DirectoryAttributes) error {
				restored[string(p)] = attrs
				order = append(order, string(p))
				return nil
			},
		}

		cleaner := newCleaner(mock)
		// the cascade takes the nested folder and then its parent
		cleaner.executeCleanup(nested, "file.txt")
		cleaner.executeCleanup(ancestor, "abc")

		// only the nested folder is written to
		mock.countFn = func(util.FullPath) (int, error) { return 1, nil }
		cleaner.OnCreateEvent(nested, "obj", false)
		cleaner.processCleanupQueue()

		if len(order) != 2 || order[0] != ancestor {
			t.Fatalf("the ancestor should be rebuilt first, got %v", order)
		}
		if got := restored[ancestor]; got.Mode != 0700 || got.Uid != 4242 {
			t.Errorf("ancestor should keep its own attributes, got mode %o uid %d", got.Mode, got.Uid)
		}
	})

	t.Run("a delete that reports failure is still observed", func(t *testing.T) {
		// the redis stores drop the folder before its parent-list member, so a failure
		// return is not proof that the folder survived
		mock := &mockFilerOps{
			countFn:  func(util.FullPath) (int, error) { return 0, nil },
			deleteFn: func(util.FullPath) error { return errors.New("partial delete") },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "file.txt")
		if len(cleaner.deleted) != 1 {
			t.Fatalf("a failed delete should still leave the folder observed, got %d", len(cleaner.deleted))
		}
	})
}

func Test_autoRemoveEmptyFoldersEnabled(t *testing.T) {
	tests := []struct {
		name      string
		attrs     map[string][]byte
		enabled   bool
		attrValue string
	}{
		{
			name:      "no attrs defaults enabled",
			attrs:     nil,
			enabled:   true,
			attrValue: "<no_attrs>",
		},
		{
			name:      "missing key defaults enabled",
			attrs:     map[string][]byte{},
			enabled:   true,
			attrValue: "<missing>",
		},
		{
			name: "allow-empty disables cleanup",
			attrs: map[string][]byte{
				s3_constants.ExtAllowEmptyFolders: []byte("true"),
			},
			enabled:   false,
			attrValue: "true",
		},
		{
			name: "explicit false keeps cleanup enabled",
			attrs: map[string][]byte{
				s3_constants.ExtAllowEmptyFolders: []byte("false"),
			},
			enabled:   true,
			attrValue: "false",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			enabled, attrValue := autoRemoveEmptyFoldersEnabled(tt.attrs)
			if enabled != tt.enabled {
				t.Fatalf("expected enabled=%v, got %v", tt.enabled, enabled)
			}
			if attrValue != tt.attrValue {
				t.Fatalf("expected attrValue=%q, got %q", tt.attrValue, attrValue)
			}
		})
	}
}

func TestEmptyFolderCleaner_ownsFolder(t *testing.T) {
	// Create a LockRing with multiple servers
	lockRing := lock_manager.NewLockRing(5 * time.Second)

	servers := []pb.ServerAddress{
		"filer1:8888",
		"filer2:8888",
		"filer3:8888",
	}
	lockRing.SetSnapshot(servers, 0)

	// Create cleaner for filer1
	cleaner1 := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer1:8888",
	}

	// Create cleaner for filer2
	cleaner2 := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer2:8888",
	}

	// Create cleaner for filer3
	cleaner3 := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer3:8888",
	}

	// Test that exactly one filer owns each folder
	testFolders := []string{
		"/buckets/mybucket/folder1",
		"/buckets/mybucket/folder2",
		"/buckets/mybucket/folder3",
		"/buckets/mybucket/a/b/c",
		"/buckets/otherbucket/x",
	}

	for _, folder := range testFolders {
		ownCount := 0
		if cleaner1.ownsFolder(folder) {
			ownCount++
		}
		if cleaner2.ownsFolder(folder) {
			ownCount++
		}
		if cleaner3.ownsFolder(folder) {
			ownCount++
		}

		if ownCount != 1 {
			t.Errorf("folder %q owned by %d filers, expected exactly 1", folder, ownCount)
		}
	}
}

func TestEmptyFolderCleaner_ownsFolder_singleServer(t *testing.T) {
	// Create a LockRing with a single server
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer1:8888",
	}

	// Single filer should own all folders
	testFolders := []string{
		"/buckets/mybucket/folder1",
		"/buckets/mybucket/folder2",
		"/buckets/otherbucket/x",
	}

	for _, folder := range testFolders {
		if !cleaner.ownsFolder(folder) {
			t.Errorf("single filer should own folder %q", folder)
		}
	}
}

func TestEmptyFolderCleaner_ownsFolder_emptyRing(t *testing.T) {
	// Create an empty LockRing
	lockRing := lock_manager.NewLockRing(5 * time.Second)

	cleaner := &EmptyFolderCleaner{
		lockRing: lockRing,
		host:     "filer1:8888",
	}

	// With empty ring, should own all folders
	if !cleaner.ownsFolder("/buckets/mybucket/folder") {
		t.Error("should own folder with empty ring")
	}
}

func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanup(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"
	now := time.Now()

	// Simulate delete event
	cleaner.OnDeleteEvent(folder, "file.txt", false, now)

	// Check that cleanup is queued
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Errorf("expected 1 pending cleanup, got %d", cleaner.GetPendingCleanupCount())
	}

	// Simulate create event
	cleaner.OnCreateEvent(folder, "newfile.txt", false)

	// Check that cleanup is cancelled
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Errorf("expected 0 pending cleanups after create, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_deduplication(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"
	now := time.Now()

	// Simulate multiple delete events for same folder
	for i := 0; i < 5; i++ {
		cleaner.OnDeleteEvent(folder, "file"+string(rune('0'+i))+".txt", false, now.Add(time.Duration(i)*time.Second))
	}

	// Check that only 1 cleanup is queued (deduplicated)
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Errorf("expected 1 pending cleanup after deduplication, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_multipleFolders(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()

	// Delete files in different folders
	cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file.txt", false, now)
	cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file.txt", false, now.Add(1*time.Second))
	cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file.txt", false, now.Add(2*time.Second))

	// Each folder should be queued
	if cleaner.GetPendingCleanupCount() != 3 {
		t.Errorf("expected 3 pending cleanups, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_notOwner(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888", "filer2:8888"}, 0)

	// Create cleaner for filer that doesn't own the folder
	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()

	// Try many folders, looking for one that filer1 doesn't own
	foundNonOwned := false
	for i := 0; i < 100; i++ {
		folder := "/buckets/mybucket/folder" + string(rune('0'+i%10)) + string(rune('0'+i/10))
		if !cleaner.ownsFolder(folder) {
			// This folder is not owned by filer1
			cleaner.OnDeleteEvent(folder, "file.txt", false, now)
			if cleaner.GetPendingCleanupCount() != 0 {
				t.Errorf("non-owner should not queue cleanup for folder %s", folder)
			}
			foundNonOwned = true
			break
		}
	}

	if !foundNonOwned {
		t.Skip("could not find a folder not owned by filer1")
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_disabled(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      false, // Disabled
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"
	now := time.Now()

	// Simulate delete event
	cleaner.OnDeleteEvent(folder, "file.txt", false, now)

	// Check that no cleanup is queued when disabled
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Errorf("disabled cleaner should not queue cleanup, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_OnDeleteEvent_directoryDeletion(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"
	now := time.Now()

	// Simulate directory delete event - should trigger cleanup
	// because subdirectory deletion also makes parent potentially empty
	cleaner.OnDeleteEvent(folder, "subdir", true, now)

	// Check that cleanup IS queued for directory deletion
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Errorf("directory deletion should trigger cleanup, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_cachedCounts(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/testfolder"

	// Initialize cached count
	cleaner.folderCounts[folder] = &folderState{roughCount: 5}

	// Simulate create events
	cleaner.OnCreateEvent(folder, "newfile1.txt", false)
	cleaner.OnCreateEvent(folder, "newfile2.txt", false)

	// Check cached count increased
	count, exists := cleaner.GetCachedFolderCount(folder)
	if !exists {
		t.Error("cached folder count should exist")
	}
	if count != 7 {
		t.Errorf("expected cached count 7, got %d", count)
	}

	// Simulate delete events
	now := time.Now()
	cleaner.OnDeleteEvent(folder, "file1.txt", false, now)
	cleaner.OnDeleteEvent(folder, "file2.txt", false, now.Add(1*time.Second))

	// Check cached count decreased
	count, exists = cleaner.GetCachedFolderCount(folder)
	if !exists {
		t.Error("cached folder count should exist")
	}
	if count != 5 {
		t.Errorf("expected cached count 5, got %d", count)
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_Stop(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()

	// Queue some cleanups
	cleaner.OnDeleteEvent("/buckets/mybucket/folder1", "file1.txt", false, now)
	cleaner.OnDeleteEvent("/buckets/mybucket/folder2", "file2.txt", false, now.Add(1*time.Second))
	cleaner.OnDeleteEvent("/buckets/mybucket/folder3", "file3.txt", false, now.Add(2*time.Second))

	// Verify cleanups are queued
	if cleaner.GetPendingCleanupCount() < 1 {
		t.Error("expected at least 1 pending cleanup before stop")
	}

	// Stop the cleaner
	cleaner.Stop()

	// Verify all cleanups are cancelled
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Errorf("expected 0 pending cleanups after stop, got %d", cleaner.GetPendingCleanupCount())
	}
}

func TestEmptyFolderCleaner_cacheEviction(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		cacheExpiry:  100 * time.Millisecond, // Short expiry for testing
		stopCh:       make(chan struct{}),
	}

	folder1 := "/buckets/mybucket/folder1"
	folder2 := "/buckets/mybucket/folder2"
	folder3 := "/buckets/mybucket/folder3"

	// Add some cache entries with old timestamps
	oldTime := time.Now().Add(-1 * time.Hour)
	cleaner.folderCounts[folder1] = &folderState{roughCount: 5, lastCheck: oldTime}
	cleaner.folderCounts[folder2] = &folderState{roughCount: 3, lastCheck: oldTime}
	// folder3 has recent activity
	cleaner.folderCounts[folder3] = &folderState{roughCount: 2, lastCheck: time.Now()}

	// Verify all entries exist
	if len(cleaner.folderCounts) != 3 {
		t.Errorf("expected 3 cache entries, got %d", len(cleaner.folderCounts))
	}

	// Run eviction
	cleaner.evictStaleCacheEntries()

	// Verify stale entries are evicted
	if len(cleaner.folderCounts) != 1 {
		t.Errorf("expected 1 cache entry after eviction, got %d", len(cleaner.folderCounts))
	}

	// Verify the recent entry still exists
	if _, exists := cleaner.folderCounts[folder3]; !exists {
		t.Error("expected folder3 to still exist in cache")
	}

	// Verify stale entries are removed
	if _, exists := cleaner.folderCounts[folder1]; exists {
		t.Error("expected folder1 to be evicted")
	}
	if _, exists := cleaner.folderCounts[folder2]; exists {
		t.Error("expected folder2 to be evicted")
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_cacheEviction_skipsEntriesInQueue(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		cacheExpiry:  100 * time.Millisecond,
		stopCh:       make(chan struct{}),
	}

	folder := "/buckets/mybucket/folder"
	oldTime := time.Now().Add(-1 * time.Hour)

	// Add a stale cache entry
	cleaner.folderCounts[folder] = &folderState{roughCount: 0, lastCheck: oldTime}
	// Also add to cleanup queue
	cleaner.cleanupQueue.Add(folder, "item", time.Now())

	// Run eviction
	cleaner.evictStaleCacheEntries()

	// Verify entry is NOT evicted because it's in cleanup queue
	if _, exists := cleaner.folderCounts[folder]; !exists {
		t.Error("expected folder to still exist in cache (is in cleanup queue)")
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_queueFIFOOrder(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		lockRing:     lockRing,
		host:         "filer1:8888",
		bucketPath:   "/buckets",
		enabled:      true,
		folderCounts: make(map[string]*folderState),
		cleanupQueue: NewCleanupQueue(1000, 10*time.Minute),
		stopCh:       make(chan struct{}),
	}

	now := time.Now()

	// Add folders in order
	folders := []string{
		"/buckets/mybucket/folder1",
		"/buckets/mybucket/folder2",
		"/buckets/mybucket/folder3",
	}
	for i, folder := range folders {
		cleaner.OnDeleteEvent(folder, "file.txt", false, now.Add(time.Duration(i)*time.Second))
	}

	// Verify queue length
	if cleaner.GetPendingCleanupCount() != 3 {
		t.Errorf("expected 3 queued folders, got %d", cleaner.GetPendingCleanupCount())
	}

	// Verify time-sorted order by popping
	for i, expected := range folders {
		folder, _, ok := cleaner.cleanupQueue.Pop()
		if !ok || folder != expected {
			t.Errorf("expected folder %s at index %d, got %s", expected, i, folder)
		}
	}

	cleaner.Stop()
}

func TestEmptyFolderCleaner_processCleanupQueue_onlyProcessesAgedItems(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	var deleted []string
	mock := &mockFilerOps{
		countFn: func(_ util.FullPath) (int, error) {
			return 0, nil
		},
		deleteFn: func(path util.FullPath) error {
			deleted = append(deleted, string(path))
			return nil
		},
	}

	maxAge := 100 * time.Millisecond
	cleaner := &EmptyFolderCleaner{
		filer:          mock,
		lockRing:       lockRing,
		host:           "filer1:8888",
		bucketPath:     "/buckets",
		enabled:        true,
		folderCounts:   make(map[string]*folderState),
		cleanupQueue:   NewCleanupQueue(1000, maxAge),
		maxCountCheck:  1000,
		cacheExpiry:    time.Minute,
		processorSleep: time.Second,
		stopCh:         make(chan struct{}),
	}

	// Add old items (well past maxAge) and a fresh item
	old := time.Now().Add(-time.Second)
	cleaner.cleanupQueue.Add("/buckets/test/folder1", "i1", old)
	cleaner.cleanupQueue.Add("/buckets/test/folder2", "i2", old.Add(time.Millisecond))
	cleaner.cleanupQueue.Add("/buckets/test/folder3", "i3", time.Now()) // fresh, should NOT be processed

	cleaner.processCleanupQueue()

	// Only the two old items should have been processed
	if len(deleted) != 2 {
		t.Fatalf("expected 2 deleted folders (aged items only), got %d: %v", len(deleted), deleted)
	}
	if got := cleaner.cleanupQueue.Len(); got != 1 {
		t.Fatalf("expected 1 item remaining in queue, got %d", got)
	}
}

func TestEmptyFolderCleaner_executeCleanup_bucketPolicyDisabledSkips(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	var deleted []string
	mock := &mockFilerOps{
		countFn: func(_ util.FullPath) (int, error) {
			return 0, nil
		},
		deleteFn: func(path util.FullPath) error {
			deleted = append(deleted, string(path))
			return nil
		},
		attrsFn: func(path util.FullPath) (map[string][]byte, error) {
			if string(path) == "/buckets/test" {
				return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("true")}, nil
			}
			return nil, nil
		},
	}

	cleaner := &EmptyFolderCleaner{
		filer:          mock,
		lockRing:       lockRing,
		host:           "filer1:8888",
		bucketPath:     "/buckets",
		enabled:        true,
		folderCounts:   make(map[string]*folderState),
		cleanupQueue:   NewCleanupQueue(1000, time.Minute),
		maxCountCheck:  1000,
		cacheExpiry:    time.Minute,
		processorSleep: time.Second,
		stopCh:         make(chan struct{}),
	}

	folder := "/buckets/test/folder"
	cleaner.executeCleanup(folder, "triggered_item")

	if len(deleted) != 0 {
		t.Fatalf("expected folder %s to be skipped, got deletions %v", folder, deleted)
	}
}

func TestEmptyFolderCleaner_InvalidateBucketPolicy(t *testing.T) {
	cleaner := &EmptyFolderCleaner{
		bucketPath: "/buckets",
		enabled:    true,
		bucketCleanupPolicies: map[string]*bucketCleanupPolicyState{
			"/buckets/test": {},
		},
	}

	cleaner.InvalidateBucketPolicy("/buckets", "test", true)
	if _, found := cleaner.bucketCleanupPolicies["/buckets/test"]; found {
		t.Fatal("expected cached bucket policy to be evicted")
	}

	cleaner.bucketCleanupPolicies["/buckets/test"] = &bucketCleanupPolicyState{}
	cleaner.InvalidateBucketPolicy("/buckets/test", "dir", true)
	cleaner.InvalidateBucketPolicy("/buckets", "test", false)
	if _, found := cleaner.bucketCleanupPolicies["/buckets/test"]; !found {
		t.Fatal("expected unrelated updates to keep the cached policy")
	}
}

func TestEmptyFolderCleaner_getBucketCleanupPolicy_concurrentUpdate(t *testing.T) {
	var cleaner *EmptyFolderCleaner
	calls := 0
	mock := &mockFilerOps{
		attrsFn: func(_ util.FullPath) (map[string][]byte, error) {
			calls++
			if calls == 1 {
				// a toggle lands while the stale read is in flight
				cleaner.InvalidateBucketPolicy("/buckets", "test", true)
				return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("true")}, nil
			}
			return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("false")}, nil
		},
	}

	cleaner = &EmptyFolderCleaner{
		filer:                 mock,
		bucketPath:            "/buckets",
		enabled:               true,
		bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
	}

	_, autoRemove, _, attrValue, err := cleaner.getBucketCleanupPolicy(context.Background(), "/buckets/test/folder")
	if err != nil {
		t.Fatalf("getBucketCleanupPolicy: %v", err)
	}
	if !autoRemove || attrValue != "false" {
		t.Fatalf("expected the post-update policy autoRemove=true attr=false, got autoRemove=%v attr=%q", autoRemove, attrValue)
	}
	if calls < 2 {
		t.Fatalf("expected the stale load to be retried, got %d attribute reads", calls)
	}
	if cached := cleaner.bucketCleanupPolicies["/buckets/test"]; cached == nil || cached.attrValue != "false" {
		t.Fatalf("expected the fresh policy to be cached, got %+v", cached)
	}
}

func TestEmptyFolderCleaner_getBucketCleanupPolicy_evictionKeepsInflightRead(t *testing.T) {
	var cleaner *EmptyFolderCleaner
	started := make(chan struct{})
	release := make(chan struct{})
	calls := 0
	mock := &mockFilerOps{
		attrsFn: func(_ util.FullPath) (map[string][]byte, error) {
			calls++
			if calls == 1 {
				close(started)
				<-release
				return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("true")}, nil
			}
			return map[string][]byte{s3_constants.ExtAllowEmptyFolders: []byte("false")}, nil
		},
	}

	cleaner = &EmptyFolderCleaner{
		filer:                 mock,
		bucketPath:            "/buckets",
		enabled:               true,
		cacheExpiry:           time.Minute,
		folderCounts:          make(map[string]*folderState),
		bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
		policyGen:             make(map[string]*bucketPolicyGen),
		cleanupQueue:          NewCleanupQueue(1000, time.Minute),
	}

	go func() {
		<-started
		cleaner.InvalidateBucketPolicy("/buckets", "test", true)
		cleaner.mu.Lock()
		cleaner.policyGen["/buckets/test"].lastBump = time.Now().Add(-time.Hour)
		cleaner.mu.Unlock()
		cleaner.evictStaleCacheEntries()
		close(release)
	}()

	_, autoRemove, _, attrValue, err := cleaner.getBucketCleanupPolicy(context.Background(), "/buckets/test/folder")
	if err != nil {
		t.Fatalf("getBucketCleanupPolicy: %v", err)
	}
	if !autoRemove || attrValue != "false" {
		t.Fatalf("expected the stalled read to be discarded, got autoRemove=%v attr=%q", autoRemove, attrValue)
	}
	if calls < 2 {
		t.Fatalf("expected the stale load to be retried, got %d attribute reads", calls)
	}
}

func TestEmptyFolderCleaner_executeCleanup_policyFailureRequeues(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	mock := &mockFilerOps{
		attrsFn: func(_ util.FullPath) (map[string][]byte, error) {
			return nil, errors.New("attrs unavailable")
		},
	}

	cleaner := &EmptyFolderCleaner{
		filer:                 mock,
		lockRing:              lockRing,
		host:                  "filer1:8888",
		bucketPath:            "/buckets",
		enabled:               true,
		folderCounts:          make(map[string]*folderState),
		bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
		cleanupQueue:          NewCleanupQueue(1000, time.Millisecond),
		stopCh:                make(chan struct{}),
	}

	folder := "/buckets/test/folder"
	for i := 0; i < DefaultMaxPolicyFailures; i++ {
		cleaner.executeCleanup(folder, "triggered_item")
		popped, _, ok := cleaner.cleanupQueue.Pop()
		if !ok || popped != folder {
			t.Fatalf("failure %d of %d should requeue the folder", i+1, DefaultMaxPolicyFailures)
		}
	}

	cleaner.executeCleanup(folder, "triggered_item")

	if got := cleaner.folderCounts[folder].policyFailures; got != DefaultMaxPolicyFailures+1 {
		t.Fatalf("policyFailures = %d, want %d", got, DefaultMaxPolicyFailures+1)
	}
	if got := cleaner.cleanupQueue.Len(); got != 0 {
		t.Fatalf("expected retries to stop after %d failures, got %d queued", DefaultMaxPolicyFailures, got)
	}
}

func TestEmptyFolderCleaner_executeCleanup_directoryMarker(t *testing.T) {
	testCases := []struct {
		name           string
		isDirKeyObj    bool
		expectDeletion bool
	}{
		{
			name:           "skips explicit directory marker",
			isDirKeyObj:    true,
			expectDeletion: false,
		},
		{
			name:           "deletes implicit empty folder",
			isDirKeyObj:    false,
			expectDeletion: true,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			lockRing := lock_manager.NewLockRing(5 * time.Second)
			lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

			var deleted []string
			mock := &mockFilerOps{
				countFn: func(_ util.FullPath) (int, error) {
					return 0, nil
				},
				deleteFn: func(path util.FullPath) error {
					deleted = append(deleted, string(path))
					return nil
				},
				isDirKeyObjFn: func(path util.FullPath) (bool, error) {
					return tc.isDirKeyObj, nil
				},
			}

			cleaner := &EmptyFolderCleaner{
				filer:          mock,
				lockRing:       lockRing,
				host:           "filer1:8888",
				bucketPath:     "/buckets",
				enabled:        true,
				folderCounts:   make(map[string]*folderState),
				cleanupQueue:   NewCleanupQueue(1000, time.Minute),
				maxCountCheck:  1000,
				cacheExpiry:    time.Minute,
				processorSleep: time.Second,
				stopCh:         make(chan struct{}),
			}

			folder := "/buckets/test/folder"
			cleaner.executeCleanup(folder, "triggered_item")

			if tc.expectDeletion {
				if len(deleted) != 1 || deleted[0] != folder {
					t.Fatalf("expected implicit empty folder %s to be deleted, got deletions %v", folder, deleted)
				}
			} else {
				if len(deleted) != 0 {
					t.Fatalf("expected explicit directory marker %s to be preserved, got deletions %v", folder, deleted)
				}
			}
		})
	}
}

func TestEmptyFolderCleaner_OnCreateEvent_cancelsCleanupForRecreatedDirectory(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	cleaner := &EmptyFolderCleaner{
		filer:          &mockFilerOps{},
		lockRing:       lockRing,
		host:           "filer1:8888",
		bucketPath:     "/buckets",
		enabled:        true,
		folderCounts:   make(map[string]*folderState),
		deleted:        make(map[string]*deletedFolder),
		cleanupQueue:   NewCleanupQueue(1000, time.Minute),
		maxCountCheck:  1000,
		cacheExpiry:    time.Minute,
		processorSleep: time.Second,
		stopCh:         make(chan struct{}),
	}

	// Dropping the table queues the folder its children were removed from, then a
	// rename puts a new table back under the same name.
	cleaner.OnDeleteEvent("/buckets/warehouse/pedsnet/person", "metadata", true, time.Now())
	if cleaner.GetPendingCleanupCount() != 1 {
		t.Fatalf("expected the emptied folder to be queued, got %d", cleaner.GetPendingCleanupCount())
	}

	cleaner.OnCreateEvent("/buckets/warehouse/pedsnet", "person", true)
	if cleaner.GetPendingCleanupCount() != 0 {
		t.Fatalf("expected the recreated folder to leave the queue, got %d", cleaner.GetPendingCleanupCount())
	}
}

func TestEmptyFolderCleaner_executeCleanup_skipsCatalogEntry(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	const namespace = "/buckets/warehouse/pedsnet"
	const table = namespace + "/person"

	var deleted []string
	mock := &mockFilerOps{
		countFn: func(_ util.FullPath) (int, error) {
			return 0, nil
		},
		deleteFn: func(path util.FullPath) error {
			deleted = append(deleted, string(path))
			return nil
		},
		attrsFn: func(path util.FullPath) (map[string][]byte, error) {
			if path == namespace || path == table {
				return map[string][]byte{s3_constants.ExtS3TablesPrefix + "metadata": []byte("{}")}, nil
			}
			return nil, nil
		},
	}

	cleaner := &EmptyFolderCleaner{
		filer:          mock,
		lockRing:       lockRing,
		host:           "filer1:8888",
		bucketPath:     "/buckets",
		enabled:        true,
		folderCounts:   make(map[string]*folderState),
		cleanupQueue:   NewCleanupQueue(1000, time.Minute),
		maxCountCheck:  1000,
		cacheExpiry:    time.Minute,
		processorSleep: time.Second,
		stopCh:         make(chan struct{}),
	}

	// The dropped table left its metadata folder empty; the cascade up from it must
	// stop at the table the rename put back, and at the namespace above it.
	cleaner.executeCleanup(table+"/metadata", "v2.metadata.json")

	if len(deleted) != 1 || deleted[0] != table+"/metadata" {
		t.Fatalf("expected only %s to be deleted, got %v", table+"/metadata", deleted)
	}
}

func TestEmptyFolderCleaner_restoreIfWrittenTo(t *testing.T) {
	lockRing := lock_manager.NewLockRing(5 * time.Second)
	lockRing.SetSnapshot([]pb.ServerAddress{"filer1:8888"}, 0)

	newCleaner := func(mock *mockFilerOps) *EmptyFolderCleaner {
		return &EmptyFolderCleaner{
			filer:                 mock,
			lockRing:              lockRing,
			host:                  "filer1:8888",
			bucketPath:            "/buckets",
			enabled:               true,
			maxCountCheck:         DefaultMaxCountCheck,
			cacheExpiry:           DefaultCacheExpiry,
			folderCounts:          make(map[string]*folderState),
			bucketCleanupPolicies: make(map[string]*bucketCleanupPolicyState),
			deleted:               make(map[string]*deletedFolder),
			cleanupQueue:          NewCleanupQueue(1000, 10*time.Minute),
			stopCh:                make(chan struct{}),
		}
	}

	const folder = "/buckets/mybucket/a/b"
	const parent = "/buckets/mybucket/a"

	t.Run("an entry that landed during the delete is restored without waiting for its event", func(t *testing.T) {
		counts := map[string]int{}
		var deleted, restored []string
		mock := &mockFilerOps{
			countFn: func(p util.FullPath) (int, error) { return counts[string(p)], nil },
			deleteFn: func(p util.FullPath) error {
				deleted = append(deleted, string(p))
				if string(p) == folder {
					// the racing write lands between the delete's own emptiness
					// check and the removal of the folder entry
					counts[folder] = 1
				}
				return nil
			},
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "obj")

		if len(restored) != 1 || restored[0] != folder {
			t.Fatalf("folder written to during the delete should be restored, got %v", restored)
		}
		if len(deleted) != 1 || deleted[0] != folder {
			t.Fatalf("a restored folder leaves the parent non-empty, so it must not cascade, got %v", deleted)
		}
		if len(cleaner.deleted) != 0 {
			t.Fatalf("a restored folder should stop being observed, got %d", len(cleaner.deleted))
		}
	})

	t.Run("a folder still empty after the delete is not restored and the parent is cascaded", func(t *testing.T) {
		var deleted, restored []string
		mock := &mockFilerOps{
			countFn:     func(util.FullPath) (int, error) { return 0, nil },
			deleteFn:    func(p util.FullPath) error { deleted = append(deleted, string(p)); return nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "obj")

		if len(restored) != 0 {
			t.Fatalf("an empty folder should not be restored, got %v", restored)
		}
		if len(deleted) != 2 || deleted[0] != folder || deleted[1] != parent {
			t.Fatalf("the parent should still be cascaded, got %v", deleted)
		}
	})

	t.Run("a restore that fails is left marked for the deferred pass", func(t *testing.T) {
		counts := map[string]int{}
		mock := &mockFilerOps{
			countFn: func(p util.FullPath) (int, error) { return counts[string(p)], nil },
			deleteFn: func(p util.FullPath) error {
				if string(p) == folder {
					counts[folder] = 1
				}
				return nil
			},
			ensureDirFn: func(util.FullPath, DirectoryAttributes) error { return errors.New("store unavailable") },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "obj")

		observed, found := cleaner.deleted[folder]
		if !found {
			t.Fatal("a folder whose restore failed must stay under observation")
		}
		if !observed.writtenTo {
			t.Fatal("a folder seen to hold an entry must be marked, or the deferred pass skips it")
		}
	})

	t.Run("a re-check that fails leaves the folder to the event path", func(t *testing.T) {
		var restored []string
		deletedFolder := false
		mock := &mockFilerOps{
			countFn: func(util.FullPath) (int, error) {
				if deletedFolder {
					return 0, errors.New("store unavailable")
				}
				return 0, nil
			},
			deleteFn:    func(p util.FullPath) error { deletedFolder = string(p) == folder; return nil },
			ensureDirFn: func(p util.FullPath, _ DirectoryAttributes) error { restored = append(restored, string(p)); return nil },
		}

		cleaner := newCleaner(mock)
		cleaner.executeCleanup(folder, "obj")

		if len(restored) != 0 {
			t.Fatalf("a failed re-check should not restore, got %v", restored)
		}
		if _, found := cleaner.deleted[folder]; !found {
			t.Fatal("a folder whose re-check failed must stay under observation")
		}
	})
}