Skip to content

Modern Docker Management, Designed for Everyone

fix: project max depth not working for filesystem discovery (#2325)

Kyle Mendell6f7470980417
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 53f6ae030..a23d44ac8 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -34,7 +34,7 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedul
scheduledPruneJob := pkg_scheduler.NewScheduledPruneJob(appServices.System, appServices.Settings, appServices.Notification)
newScheduler.RegisterJob(scheduledPruneJob)
- fsWatcherJob, err := pkg_scheduler.RegisterFilesystemWatcherJob(appCtx, appServices.Project, appServices.Template, appServices.Settings)
+ fsWatcherJob, err := pkg_scheduler.RegisterFilesystemWatcherJob(appCtx, appServices.Project, appServices.Template, appServices.Settings, appConfig.ProjectScanMaxDepth)
if err != nil {
slog.ErrorContext(appCtx, "Failed to register filesystem watcher job", "error", err)
}
diff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.go
index b2d22b780..da6dae802 100644
--- a/backend/internal/services/project_service.go
+++ b/backend/internal/services/project_service.go
@@ -903,7 +903,7 @@ func (s *ProjectService) SyncProjectsFromFileSystem(ctx context.Context) error {
return nil
}
- discoveredProjects, discoveryErr := projects.DiscoverProjectDirectories(projectsDir, followProjectSymlinks)
+ discoveredProjects, discoveryErr := projects.DiscoverProjectDirectories(projectsDir, followProjectSymlinks, s.config.ProjectScanMaxDepth)
if discoveryErr != nil {
if os.IsNotExist(discoveryErr) {
return nil
@@ -1088,7 +1088,7 @@ func (s *ProjectService) countProjectFolders(ctx context.Context) (int, error) {
return 0, nil
}
- discoveredProjects, discoveryErr := projects.DiscoverProjectDirectories(projectsDir, followProjectSymlinks)
+ discoveredProjects, discoveryErr := projects.DiscoverProjectDirectories(projectsDir, followProjectSymlinks, s.config.ProjectScanMaxDepth)
if discoveryErr != nil {
return 0, fmt.Errorf("failed to discover project directories in %s: %w", projectsDir, discoveryErr)
}
diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.go
index 5d5f024e0..c96f9c46d 100644
--- a/backend/internal/services/project_service_test.go
+++ b/backend/internal/services/project_service_test.go
@@ -1786,6 +1786,30 @@ func TestProjectService_SyncProjectsFromFileSystem_DiscoversNestedProjectsAndRel
assert.Equal(t, "project2", items[1].DirName)
}
+func TestProjectService_SyncProjectsFromFileSystem_RespectsConfiguredScanMaxDepth(t *testing.T) {
+ db := setupProjectTestDB(t)
+ ctx := context.Background()
+
+ settingsService, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ projectsRoot := t.TempDir()
+ topLevelPath := createComposeProjectDir(t, projectsRoot, "project1")
+ createComposeProjectDir(t, projectsRoot, filepath.Join("group", "project2"))
+
+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
+ t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")
+
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())
+ require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
+
+ items, err := svc.ListAllProjects(ctx)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ assert.Equal(t, "project1", items[0].Name)
+ assert.Equal(t, topLevelPath, items[0].Path)
+}
+
func TestProjectService_ListProjects_LoadsProjectIconFromGlobalEnvInIncludedMetadata(t *testing.T) {
db := setupProjectTestDB(t)
ctx := context.Background()
@@ -1855,6 +1879,27 @@ func TestProjectService_CountProjectFolders_RecursivelyCountsNestedProjects(t *t
assert.Equal(t, 3, count)
}
+func TestProjectService_CountProjectFolders_RespectsConfiguredScanMaxDepth(t *testing.T) {
+ db := setupProjectTestDB(t)
+ ctx := context.Background()
+
+ settingsService, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ projectsRoot := t.TempDir()
+ createComposeProjectDir(t, projectsRoot, "project1")
+ createComposeProjectDir(t, projectsRoot, filepath.Join("group", "project2"))
+
+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
+ t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")
+
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())
+
+ count, err := svc.countProjectFolders(ctx)
+ require.NoError(t, err)
+ assert.Equal(t, 1, count)
+}
+
func TestProjectService_SyncProjectsFromFileSystem_RemovesDeletedNestedProject(t *testing.T) {
db := setupProjectTestDB(t)
ctx := context.Background()
diff --git a/backend/pkg/fswatch/watcher_test.go b/backend/pkg/fswatch/watcher_test.go
index b7b0ed7ef..d352a3999 100644
--- a/backend/pkg/fswatch/watcher_test.go
+++ b/backend/pkg/fswatch/watcher_test.go
@@ -241,6 +241,72 @@ func TestWatcher_StartTriggersOnNestedProjectFileBeyondDepthOne(t *testing.T) {
}
}
+func TestWatcher_StartDoesNotTriggerOnNestedProjectFileBeyondConfiguredDepth(t *testing.T) {
+ root := t.TempDir()
+ nestedDir := filepath.Join(root, "main-project", "sub-project1")
+ require.NoError(t, os.MkdirAll(nestedDir, 0o755))
+
+ changeCh := make(chan struct{}, 1)
+ ctx := t.Context()
+
+ watcher, err := NewWatcher(root, WatcherOptions{
+ Debounce: 25 * time.Millisecond,
+ MaxDepth: 1,
+ OnChange: func(context.Context) {
+ select {
+ case changeCh <- struct{}{}:
+ default:
+ }
+ },
+ })
+ require.NoError(t, err)
+ require.NoError(t, watcher.Start(ctx))
+ defer func() {
+ require.NoError(t, watcher.Stop())
+ }()
+
+ require.NoError(t, os.WriteFile(filepath.Join(nestedDir, "compose.yaml"), []byte("services: {}\n"), 0o644))
+
+ select {
+ case <-changeCh:
+ t.Fatal("did not expect nested compose file beyond max depth to trigger watcher callback")
+ case <-time.After(500 * time.Millisecond):
+ }
+}
+
+func TestWatcher_StartTriggersOnDirectChildProjectFileAtConfiguredDepth(t *testing.T) {
+ root := t.TempDir()
+ projectDir := filepath.Join(root, "main-project")
+ require.NoError(t, os.MkdirAll(projectDir, 0o755))
+
+ changeCh := make(chan struct{}, 1)
+ ctx := t.Context()
+
+ watcher, err := NewWatcher(root, WatcherOptions{
+ Debounce: 25 * time.Millisecond,
+ MaxDepth: 1,
+ OnChange: func(context.Context) {
+ select {
+ case changeCh <- struct{}{}:
+ default:
+ }
+ },
+ })
+ require.NoError(t, err)
+ require.NoError(t, watcher.Start(ctx))
+ defer func() {
+ require.NoError(t, watcher.Stop())
+ }()
+
+ require.NoError(t, os.WriteFile(filepath.Join(projectDir, "compose.yaml"), []byte("services: {}\n"), 0o644))
+
+ select {
+ case <-changeCh:
+ case <-time.After(2 * time.Second):
+ t.Fatal("expected direct child compose file at max depth to trigger watcher callback")
+ }
+}
+
func TestWatcher_StartIgnoresBareNestedDirectoryCreateAndRemove(t *testing.T) {
root := t.TempDir()
diff --git a/backend/pkg/projects/discovery.go b/backend/pkg/projects/discovery.go
index eeb3a1474..cbf045b5a 100644
--- a/backend/pkg/projects/discovery.go
+++ b/backend/pkg/projects/discovery.go
@@ -60,7 +60,7 @@ func IsProjectDirectoryPath(path string, followSymlinks bool) (bool, error) {
return resolvedInfo.IsDir(), nil
}
-func DiscoverProjectDirectories(root string, followSymlinks bool) ([]DiscoveredProjectDir, error) {
+func DiscoverProjectDirectories(root string, followSymlinks bool, maxDepth int) ([]DiscoveredProjectDir, error) {
root = filepath.Clean(root)
isDir, err := IsProjectDirectoryPath(root, followSymlinks)
@@ -74,7 +74,7 @@ func DiscoverProjectDirectories(root string, followSymlinks bool) ([]DiscoveredP
discovered := make([]DiscoveredProjectDir, 0)
ancestors := make(map[string]struct{})
- if err := walkProjectDirectoriesInternal(root, true, followSymlinks, ancestors, &discovered); err != nil {
+ if err := walkProjectDirectoriesInternal(root, true, 0, maxDepth, followSymlinks, ancestors, &discovered); err != nil {
return nil, err
}
@@ -85,7 +85,7 @@ func DiscoverProjectDirectories(root string, followSymlinks bool) ([]DiscoveredP
return discovered, nil
}
-func walkProjectDirectoriesInternal(path string, isRoot bool, followSymlinks bool, ancestors map[string]struct{}, discovered *[]DiscoveredProjectDir) error {
+func walkProjectDirectoriesInternal(path string, isRoot bool, currentDepth int, maxDepth int, followSymlinks bool, ancestors map[string]struct{}, discovered *[]DiscoveredProjectDir) error {
identity, err := ResolveDirectoryIdentityInternal(path)
if err != nil {
return err
@@ -115,6 +115,10 @@ func walkProjectDirectoriesInternal(path string, isRoot bool, followSymlinks boo
}
}
+ if maxDepth > 0 && currentDepth >= maxDepth {
+ return nil
+ }
+
entries, err := os.ReadDir(path)
if err != nil {
return err
@@ -126,7 +130,7 @@ func walkProjectDirectoriesInternal(path string, isRoot bool, followSymlinks boo
continue
}
- if err := walkProjectDirectoriesInternal(childPath, false, followSymlinks, ancestors, discovered); err != nil {
+ if err := walkProjectDirectoriesInternal(childPath, false, currentDepth+1, maxDepth, followSymlinks, ancestors, discovered); err != nil {
return err
}
}
diff --git a/backend/pkg/projects/discovery_test.go b/backend/pkg/projects/discovery_test.go
index 581fbb1d5..401ab5e79 100644
--- a/backend/pkg/projects/discovery_test.go
+++ b/backend/pkg/projects/discovery_test.go
@@ -8,15 +8,15 @@ import (
"github.com/stretchr/testify/require"
)
-// writeComposeFile creates an empty compose.yml at the given directory.
-func writeComposeFile(t *testing.T, dir string) {
+// writeComposeFileInternal creates an empty compose.yml at the given directory.
+func writeComposeFileInternal(t *testing.T, dir string) {
t.Helper()
require.NoError(t, os.MkdirAll(dir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(dir, "compose.yml"), []byte("services: {}\n"), 0o644))
}
-// discoveredNames returns the DirName of each discovered project, preserving order.
-func discoveredNames(dirs []DiscoveredProjectDir) []string {
+// discoveredNamesInternal returns the DirName of each discovered project, preserving order.
+func discoveredNamesInternal(dirs []DiscoveredProjectDir) []string {
out := make([]string, 0, len(dirs))
for _, d := range dirs {
out = append(out, d.DirName)
@@ -35,14 +35,14 @@ func TestDiscoverProjectDirectories_StopsDescentAtCompose(t *testing.T) {
// root/networking/compose.yml
// root/networking/adguardhome/compose.yml
// root/networking/nginx-proxy-manager/compose.yml
- writeComposeFile(t, filepath.Join(root, "networking"))
- writeComposeFile(t, filepath.Join(root, "networking", "adguardhome"))
- writeComposeFile(t, filepath.Join(root, "networking", "nginx-proxy-manager"))
+ writeComposeFileInternal(t, filepath.Join(root, "networking"))
+ writeComposeFileInternal(t, filepath.Join(root, "networking", "adguardhome"))
+ writeComposeFileInternal(t, filepath.Join(root, "networking", "nginx-proxy-manager"))
- discovered, err := DiscoverProjectDirectories(root, false)
+ discovered, err := DiscoverProjectDirectories(root, false, 0)
require.NoError(t, err)
- names := discoveredNames(discovered)
+ names := discoveredNamesInternal(discovered)
require.Equal(t, []string{"networking"}, names,
"only the parent compose project should be discovered; children are assumed to be included")
}
@@ -53,13 +53,13 @@ func TestDiscoverProjectDirectories_StopsDescentAtCompose(t *testing.T) {
func TestDiscoverProjectDirectories_SiblingProjectsAtRoot(t *testing.T) {
root := t.TempDir()
- writeComposeFile(t, filepath.Join(root, "app1"))
- writeComposeFile(t, filepath.Join(root, "app2"))
+ writeComposeFileInternal(t, filepath.Join(root, "app1"))
+ writeComposeFileInternal(t, filepath.Join(root, "app2"))
- discovered, err := DiscoverProjectDirectories(root, false)
+ discovered, err := DiscoverProjectDirectories(root, false, 0)
require.NoError(t, err)
- names := discoveredNames(discovered)
+ names := discoveredNamesInternal(discovered)
require.ElementsMatch(t, []string{"app1", "app2"}, names)
}
@@ -73,13 +73,13 @@ func TestDiscoverProjectDirectories_RootWithComposeAndSiblings(t *testing.T) {
// root/compose.yml (a root-level "project")
// root/app1/compose.yml
// root/app2/compose.yml
- writeComposeFile(t, root)
- writeComposeFile(t, filepath.Join(root, "app1"))
- writeComposeFile(t, filepath.Join(root, "app2"))
+ writeComposeFileInternal(t, root)
+ writeComposeFileInternal(t, filepath.Join(root, "app1"))
+ writeComposeFileInternal(t, filepath.Join(root, "app2"))
- discovered := DiscoveredProjectDirectories_call(t, root)
+ discovered := discoverProjectDirectoriesInternal(t, root)
- names := discoveredNames(discovered)
+ names := discoveredNamesInternal(discovered)
require.Len(t, names, 3)
require.Contains(t, names, "app1")
require.Contains(t, names, "app2")
@@ -94,12 +94,12 @@ func TestDiscoverProjectDirectories_NestedStandaloneProject(t *testing.T) {
root := t.TempDir()
// root/sub/nested/compose.yml (no compose file at root/sub/)
- writeComposeFile(t, filepath.Join(root, "sub", "nested"))
+ writeComposeFileInternal(t, filepath.Join(root, "sub", "nested"))
- discovered, err := DiscoverProjectDirectories(root, false)
+ discovered, err := DiscoverProjectDirectories(root, false, 0)
require.NoError(t, err)
- names := discoveredNames(discovered)
+ names := discoveredNamesInternal(discovered)
require.Equal(t, []string{"nested"}, names)
}
@@ -109,18 +109,43 @@ func TestDiscoverProjectDirectories_SupportsCustomComposeFilename(t *testing.T)
require.NoError(t, os.MkdirAll(projectDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(projectDir, "radarr.yaml"), []byte("services: {}\n"), 0o644))
- discovered, err := DiscoverProjectDirectories(root, false)
+ discovered, err := DiscoverProjectDirectories(root, false, 0)
require.NoError(t, err)
require.Len(t, discovered, 1)
require.Equal(t, "Radarr-3", discovered[0].DirName)
require.Equal(t, projectDir, discovered[0].Path)
}
-// DiscoveredProjectDirectories_call is a tiny helper so tests can share the
+// discoverProjectDirectoriesInternal is a tiny helper so tests can share the
// err-check boilerplate.
-func DiscoveredProjectDirectories_call(t *testing.T, root string) []DiscoveredProjectDir {
+func discoverProjectDirectoriesInternal(t *testing.T, root string) []DiscoveredProjectDir {
t.Helper()
- d, err := DiscoverProjectDirectories(root, false)
+ d, err := DiscoverProjectDirectories(root, false, 0)
require.NoError(t, err)
return d
}
+
+func TestDiscoverProjectDirectories_RespectsMaxDepth(t *testing.T) {
+ root := t.TempDir()
+
+ writeComposeFileInternal(t, filepath.Join(root, "top-level"))
+ writeComposeFileInternal(t, filepath.Join(root, "group", "nested"))
+
+ discovered, err := DiscoverProjectDirectories(root, false, 1)
+ require.NoError(t, err)
+
+ names := discoveredNamesInternal(discovered)
+ require.Equal(t, []string{"top-level"}, names)
+}
+
+func TestDiscoverProjectDirectories_UnlimitedDepthStillFindsNestedProject(t *testing.T) {
+ root := t.TempDir()
+
+ writeComposeFileInternal(t, filepath.Join(root, "group", "nested"))
+
+ discovered, err := DiscoverProjectDirectories(root, false, 0)
+ require.NoError(t, err)
+
+ names := discoveredNamesInternal(discovered)
+ require.Equal(t, []string{"nested"}, names)
+}
diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.go
index ee9db8627..1bcb6ef8f 100644
--- a/backend/pkg/scheduler/filesystem_watcher_job.go
+++ b/backend/pkg/scheduler/filesystem_watcher_job.go
@@ -16,6 +16,7 @@ type FilesystemWatcherJob struct {
projectService *services.ProjectService
templateService *services.TemplateService
settingsService *services.SettingsService
+ projectScanDepth int
projectsWatcher *fswatch.Watcher
templatesWatcher *fswatch.Watcher
mu sync.Mutex
@@ -25,16 +26,18 @@ func NewFilesystemWatcherJob(
projectService *services.ProjectService,
templateService *services.TemplateService,
settingsService *services.SettingsService,
+ projectScanDepth int,
) *FilesystemWatcherJob {
return &FilesystemWatcherJob{
- projectService: projectService,
- templateService: templateService,
- settingsService: settingsService,
+ projectService: projectService,
+ templateService: templateService,
+ settingsService: settingsService,
+ projectScanDepth: projectScanDepth,
}
}
-func RegisterFilesystemWatcherJob(ctx context.Context, projectService *services.ProjectService, templateService *services.TemplateService, settingsService *services.SettingsService) (*FilesystemWatcherJob, error) {
- job := NewFilesystemWatcherJob(projectService, templateService, settingsService)
+func RegisterFilesystemWatcherJob(ctx context.Context, projectService *services.ProjectService, templateService *services.TemplateService, settingsService *services.SettingsService, projectScanDepth int) (*FilesystemWatcherJob, error) {
+ job := NewFilesystemWatcherJob(projectService, templateService, settingsService, projectScanDepth)
go func() {
if err := job.Start(ctx); err != nil {
@@ -58,12 +61,7 @@ func (j *FilesystemWatcherJob) Start(ctx context.Context) error {
followProjectSymlinks := settings.FollowProjectSymlinks.IsTrue()
j.logRecursiveProjectsWatchLimitWarningInternal(ctx, projectsDirectory)
- sw, err := fswatch.NewWatcher(projectsDirectory, fswatch.WatcherOptions{
- Debounce: 3 * time.Second, // Wait 3 seconds after last change before syncing
- OnChange: j.handleFilesystemChange,
- MaxDepth: 0,
- FollowSymlinkDirs: followProjectSymlinks,
- })
+ sw, err := fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(followProjectSymlinks))
if err != nil {
return err
}
@@ -193,12 +191,7 @@ func (j *FilesystemWatcherJob) RestartProjectsWatcher(ctx context.Context) error
j.logRecursiveProjectsWatchLimitWarningInternal(ctx, projectsDirectory)
// Create a new watcher with the updated path
- sw, err := fswatch.NewWatcher(projectsDirectory, fswatch.WatcherOptions{
- Debounce: 3 * time.Second,
- OnChange: j.handleFilesystemChange,
- MaxDepth: 0,
- FollowSymlinkDirs: followProjectSymlinks,
- })
+ sw, err := fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(followProjectSymlinks))
if err != nil {
return err
}
@@ -232,3 +225,12 @@ func (j *FilesystemWatcherJob) logRecursiveProjectsWatchLimitWarningInternal(ctx
"path", projectsDirectory,
"sysctl", "fs.inotify.max_user_watches")
}
+
+func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlinks bool) fswatch.WatcherOptions {
+ return fswatch.WatcherOptions{
+ Debounce: 3 * time.Second, // Wait 3 seconds after last change before syncing
+ OnChange: j.handleFilesystemChange,
+ MaxDepth: j.projectScanDepth,
+ FollowSymlinkDirs: followProjectSymlinks,
+ }
+}
diff --git a/backend/pkg/scheduler/filesystem_watcher_job_test.go b/backend/pkg/scheduler/filesystem_watcher_job_test.go
new file mode 100644
index 000000000..e6a8a37cc
--- /dev/null
+++ b/backend/pkg/scheduler/filesystem_watcher_job_test.go
@@ -0,0 +1,18 @@
+package scheduler
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *testing.T) {
+ job := &FilesystemWatcherJob{
+ projectScanDepth: 1,
+ }
+
+ opts := job.projectWatcherOptionsInternal(true)
+
+ assert.Equal(t, 1, opts.MaxDepth)
+ assert.True(t, opts.FollowSymlinkDirs)
+}