Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: project max depth not working for filesystem discovery (#2325)
Kyle Mendell6f7470980417
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 53f6ae030..a23d44ac8 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -34,7 +34,7 @@ func registerJobs(appCtx context.Context, newScheduler *pkg_scheduler.JobSchedulscheduledPruneJob := pkg_scheduler.NewScheduledPruneJob(appServices.System, appServices.Settings, appServices.Notification)newScheduler.RegisterJob(scheduledPruneJob)- fsWatcherJob, err := pkg_scheduler.RegisterFilesystemWatcherJob(appCtx, appServices.Project, appServices.Template, appServices.Settings)+ fsWatcherJob, err := pkg_scheduler.RegisterFilesystemWatcherJob(appCtx, appServices.Project, appServices.Template, appServices.Settings, appConfig.ProjectScanMaxDepth)if err != nil {slog.ErrorContext(appCtx, "Failed to register filesystem watcher job", "error", err)}diff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.goindex b2d22b780..da6dae802 100644--- a/backend/internal/services/project_service.go+++ b/backend/internal/services/project_service.go@@ -903,7 +903,7 @@ func (s *ProjectService) SyncProjectsFromFileSystem(ctx context.Context) error {return nil}- discoveredProjects, discoveryErr := projects.DiscoverProjectDirectories(projectsDir, followProjectSymlinks)+ discoveredProjects, discoveryErr := projects.DiscoverProjectDirectories(projectsDir, followProjectSymlinks, s.config.ProjectScanMaxDepth)if discoveryErr != nil {if os.IsNotExist(discoveryErr) {return nil@@ -1088,7 +1088,7 @@ func (s *ProjectService) countProjectFolders(ctx context.Context) (int, error) {return 0, nil}- discoveredProjects, discoveryErr := projects.DiscoverProjectDirectories(projectsDir, followProjectSymlinks)+ discoveredProjects, discoveryErr := projects.DiscoverProjectDirectories(projectsDir, followProjectSymlinks, s.config.ProjectScanMaxDepth)if discoveryErr != nil {return 0, fmt.Errorf("failed to discover project directories in %s: %w", projectsDir, discoveryErr)}diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.goindex 5d5f024e0..c96f9c46d 100644--- a/backend/internal/services/project_service_test.go+++ b/backend/internal/services/project_service_test.go@@ -1786,6 +1786,30 @@ func TestProjectService_SyncProjectsFromFileSystem_DiscoversNestedProjectsAndRelassert.Equal(t, "project2", items[1].DirName)}+func TestProjectService_SyncProjectsFromFileSystem_RespectsConfiguredScanMaxDepth(t *testing.T) {+ db := setupProjectTestDB(t)+ ctx := context.Background()++ settingsService, err := NewSettingsService(ctx, db)+ require.NoError(t, err)++ projectsRoot := t.TempDir()+ topLevelPath := createComposeProjectDir(t, projectsRoot, "project1")+ createComposeProjectDir(t, projectsRoot, filepath.Join("group", "project2"))++ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))+ t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")++ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())+ require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))++ items, err := svc.ListAllProjects(ctx)+ require.NoError(t, err)+ require.Len(t, items, 1)+ assert.Equal(t, "project1", items[0].Name)+ assert.Equal(t, topLevelPath, items[0].Path)+}+func TestProjectService_ListProjects_LoadsProjectIconFromGlobalEnvInIncludedMetadata(t *testing.T) {db := setupProjectTestDB(t)ctx := context.Background()@@ -1855,6 +1879,27 @@ func TestProjectService_CountProjectFolders_RecursivelyCountsNestedProjects(t *tassert.Equal(t, 3, count)}+func TestProjectService_CountProjectFolders_RespectsConfiguredScanMaxDepth(t *testing.T) {+ db := setupProjectTestDB(t)+ ctx := context.Background()++ settingsService, err := NewSettingsService(ctx, db)+ require.NoError(t, err)++ projectsRoot := t.TempDir()+ createComposeProjectDir(t, projectsRoot, "project1")+ createComposeProjectDir(t, projectsRoot, filepath.Join("group", "project2"))++ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))+ t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")++ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())++ count, err := svc.countProjectFolders(ctx)+ require.NoError(t, err)+ assert.Equal(t, 1, count)+}+func TestProjectService_SyncProjectsFromFileSystem_RemovesDeletedNestedProject(t *testing.T) {db := setupProjectTestDB(t)ctx := context.Background()diff --git a/backend/pkg/fswatch/watcher_test.go b/backend/pkg/fswatch/watcher_test.goindex b7b0ed7ef..d352a3999 100644--- a/backend/pkg/fswatch/watcher_test.go+++ b/backend/pkg/fswatch/watcher_test.go@@ -241,6 +241,72 @@ func TestWatcher_StartTriggersOnNestedProjectFileBeyondDepthOne(t *testing.T) {}}+func TestWatcher_StartDoesNotTriggerOnNestedProjectFileBeyondConfiguredDepth(t *testing.T) {+ root := t.TempDir()+ nestedDir := filepath.Join(root, "main-project", "sub-project1")+ require.NoError(t, os.MkdirAll(nestedDir, 0o755))++ changeCh := make(chan struct{}, 1)+ ctx := t.Context()++ watcher, err := NewWatcher(root, WatcherOptions{+ Debounce: 25 * time.Millisecond,+ MaxDepth: 1,+ OnChange: func(context.Context) {+ select {+ case changeCh <- struct{}{}:+ default:+ }+ },+ })+ require.NoError(t, err)+ require.NoError(t, watcher.Start(ctx))+ defer func() {+ require.NoError(t, watcher.Stop())+ }()++ require.NoError(t, os.WriteFile(filepath.Join(nestedDir, "compose.yaml"), []byte("services: {}\n"), 0o644))++ select {+ case <-changeCh:+ t.Fatal("did not expect nested compose file beyond max depth to trigger watcher callback")+ case <-time.After(500 * time.Millisecond):+ }+}++func TestWatcher_StartTriggersOnDirectChildProjectFileAtConfiguredDepth(t *testing.T) {+ root := t.TempDir()+ projectDir := filepath.Join(root, "main-project")+ require.NoError(t, os.MkdirAll(projectDir, 0o755))++ changeCh := make(chan struct{}, 1)+ ctx := t.Context()++ watcher, err := NewWatcher(root, WatcherOptions{+ Debounce: 25 * time.Millisecond,+ MaxDepth: 1,+ OnChange: func(context.Context) {+ select {+ case changeCh <- struct{}{}:+ default:+ }+ },+ })+ require.NoError(t, err)+ require.NoError(t, watcher.Start(ctx))+ defer func() {+ require.NoError(t, watcher.Stop())+ }()++ require.NoError(t, os.WriteFile(filepath.Join(projectDir, "compose.yaml"), []byte("services: {}\n"), 0o644))++ select {+ case <-changeCh:+ case <-time.After(2 * time.Second):+ t.Fatal("expected direct child compose file at max depth to trigger watcher callback")+ }+}+func TestWatcher_StartIgnoresBareNestedDirectoryCreateAndRemove(t *testing.T) {root := t.TempDir()diff --git a/backend/pkg/projects/discovery.go b/backend/pkg/projects/discovery.goindex eeb3a1474..cbf045b5a 100644--- a/backend/pkg/projects/discovery.go+++ b/backend/pkg/projects/discovery.go@@ -60,7 +60,7 @@ func IsProjectDirectoryPath(path string, followSymlinks bool) (bool, error) {return resolvedInfo.IsDir(), nil}-func DiscoverProjectDirectories(root string, followSymlinks bool) ([]DiscoveredProjectDir, error) {+func DiscoverProjectDirectories(root string, followSymlinks bool, maxDepth int) ([]DiscoveredProjectDir, error) {root = filepath.Clean(root)isDir, err := IsProjectDirectoryPath(root, followSymlinks)@@ -74,7 +74,7 @@ func DiscoverProjectDirectories(root string, followSymlinks bool) ([]DiscoveredPdiscovered := make([]DiscoveredProjectDir, 0)ancestors := make(map[string]struct{})- if err := walkProjectDirectoriesInternal(root, true, followSymlinks, ancestors, &discovered); err != nil {+ if err := walkProjectDirectoriesInternal(root, true, 0, maxDepth, followSymlinks, ancestors, &discovered); err != nil {return nil, err}@@ -85,7 +85,7 @@ func DiscoverProjectDirectories(root string, followSymlinks bool) ([]DiscoveredPreturn discovered, nil}-func walkProjectDirectoriesInternal(path string, isRoot bool, followSymlinks bool, ancestors map[string]struct{}, discovered *[]DiscoveredProjectDir) error {+func walkProjectDirectoriesInternal(path string, isRoot bool, currentDepth int, maxDepth int, followSymlinks bool, ancestors map[string]struct{}, discovered *[]DiscoveredProjectDir) error {identity, err := ResolveDirectoryIdentityInternal(path)if err != nil {return err@@ -115,6 +115,10 @@ func walkProjectDirectoriesInternal(path string, isRoot bool, followSymlinks boo}}+ if maxDepth > 0 && currentDepth >= maxDepth {+ return nil+ }+entries, err := os.ReadDir(path)if err != nil {return err@@ -126,7 +130,7 @@ func walkProjectDirectoriesInternal(path string, isRoot bool, followSymlinks boocontinue}- if err := walkProjectDirectoriesInternal(childPath, false, followSymlinks, ancestors, discovered); err != nil {+ if err := walkProjectDirectoriesInternal(childPath, false, currentDepth+1, maxDepth, followSymlinks, ancestors, discovered); err != nil {return err}}diff --git a/backend/pkg/projects/discovery_test.go b/backend/pkg/projects/discovery_test.goindex 581fbb1d5..401ab5e79 100644--- a/backend/pkg/projects/discovery_test.go+++ b/backend/pkg/projects/discovery_test.go@@ -8,15 +8,15 @@ import ("github.com/stretchr/testify/require")-// writeComposeFile creates an empty compose.yml at the given directory.-func writeComposeFile(t *testing.T, dir string) {+// writeComposeFileInternal creates an empty compose.yml at the given directory.+func writeComposeFileInternal(t *testing.T, dir string) {t.Helper()require.NoError(t, os.MkdirAll(dir, 0o755))require.NoError(t, os.WriteFile(filepath.Join(dir, "compose.yml"), []byte("services: {}\n"), 0o644))}-// discoveredNames returns the DirName of each discovered project, preserving order.-func discoveredNames(dirs []DiscoveredProjectDir) []string {+// discoveredNamesInternal returns the DirName of each discovered project, preserving order.+func discoveredNamesInternal(dirs []DiscoveredProjectDir) []string {out := make([]string, 0, len(dirs))for _, d := range dirs {out = append(out, d.DirName)@@ -35,14 +35,14 @@ func TestDiscoverProjectDirectories_StopsDescentAtCompose(t *testing.T) {// root/networking/compose.yml// root/networking/adguardhome/compose.yml// root/networking/nginx-proxy-manager/compose.yml- writeComposeFile(t, filepath.Join(root, "networking"))- writeComposeFile(t, filepath.Join(root, "networking", "adguardhome"))- writeComposeFile(t, filepath.Join(root, "networking", "nginx-proxy-manager"))+ writeComposeFileInternal(t, filepath.Join(root, "networking"))+ writeComposeFileInternal(t, filepath.Join(root, "networking", "adguardhome"))+ writeComposeFileInternal(t, filepath.Join(root, "networking", "nginx-proxy-manager"))- discovered, err := DiscoverProjectDirectories(root, false)+ discovered, err := DiscoverProjectDirectories(root, false, 0)require.NoError(t, err)- names := discoveredNames(discovered)+ names := discoveredNamesInternal(discovered)require.Equal(t, []string{"networking"}, names,"only the parent compose project should be discovered; children are assumed to be included")}@@ -53,13 +53,13 @@ func TestDiscoverProjectDirectories_StopsDescentAtCompose(t *testing.T) {func TestDiscoverProjectDirectories_SiblingProjectsAtRoot(t *testing.T) {root := t.TempDir()- writeComposeFile(t, filepath.Join(root, "app1"))- writeComposeFile(t, filepath.Join(root, "app2"))+ writeComposeFileInternal(t, filepath.Join(root, "app1"))+ writeComposeFileInternal(t, filepath.Join(root, "app2"))- discovered, err := DiscoverProjectDirectories(root, false)+ discovered, err := DiscoverProjectDirectories(root, false, 0)require.NoError(t, err)- names := discoveredNames(discovered)+ names := discoveredNamesInternal(discovered)require.ElementsMatch(t, []string{"app1", "app2"}, names)}@@ -73,13 +73,13 @@ func TestDiscoverProjectDirectories_RootWithComposeAndSiblings(t *testing.T) {// root/compose.yml (a root-level "project")// root/app1/compose.yml// root/app2/compose.yml- writeComposeFile(t, root)- writeComposeFile(t, filepath.Join(root, "app1"))- writeComposeFile(t, filepath.Join(root, "app2"))+ writeComposeFileInternal(t, root)+ writeComposeFileInternal(t, filepath.Join(root, "app1"))+ writeComposeFileInternal(t, filepath.Join(root, "app2"))- discovered := DiscoveredProjectDirectories_call(t, root)+ discovered := discoverProjectDirectoriesInternal(t, root)- names := discoveredNames(discovered)+ names := discoveredNamesInternal(discovered)require.Len(t, names, 3)require.Contains(t, names, "app1")require.Contains(t, names, "app2")@@ -94,12 +94,12 @@ func TestDiscoverProjectDirectories_NestedStandaloneProject(t *testing.T) {root := t.TempDir()// root/sub/nested/compose.yml (no compose file at root/sub/)- writeComposeFile(t, filepath.Join(root, "sub", "nested"))+ writeComposeFileInternal(t, filepath.Join(root, "sub", "nested"))- discovered, err := DiscoverProjectDirectories(root, false)+ discovered, err := DiscoverProjectDirectories(root, false, 0)require.NoError(t, err)- names := discoveredNames(discovered)+ names := discoveredNamesInternal(discovered)require.Equal(t, []string{"nested"}, names)}@@ -109,18 +109,43 @@ func TestDiscoverProjectDirectories_SupportsCustomComposeFilename(t *testing.T)require.NoError(t, os.MkdirAll(projectDir, 0o755))require.NoError(t, os.WriteFile(filepath.Join(projectDir, "radarr.yaml"), []byte("services: {}\n"), 0o644))- discovered, err := DiscoverProjectDirectories(root, false)+ discovered, err := DiscoverProjectDirectories(root, false, 0)require.NoError(t, err)require.Len(t, discovered, 1)require.Equal(t, "Radarr-3", discovered[0].DirName)require.Equal(t, projectDir, discovered[0].Path)}-// DiscoveredProjectDirectories_call is a tiny helper so tests can share the+// discoverProjectDirectoriesInternal is a tiny helper so tests can share the// err-check boilerplate.-func DiscoveredProjectDirectories_call(t *testing.T, root string) []DiscoveredProjectDir {+func discoverProjectDirectoriesInternal(t *testing.T, root string) []DiscoveredProjectDir {t.Helper()- d, err := DiscoverProjectDirectories(root, false)+ d, err := DiscoverProjectDirectories(root, false, 0)require.NoError(t, err)return d}++func TestDiscoverProjectDirectories_RespectsMaxDepth(t *testing.T) {+ root := t.TempDir()++ writeComposeFileInternal(t, filepath.Join(root, "top-level"))+ writeComposeFileInternal(t, filepath.Join(root, "group", "nested"))++ discovered, err := DiscoverProjectDirectories(root, false, 1)+ require.NoError(t, err)++ names := discoveredNamesInternal(discovered)+ require.Equal(t, []string{"top-level"}, names)+}++func TestDiscoverProjectDirectories_UnlimitedDepthStillFindsNestedProject(t *testing.T) {+ root := t.TempDir()++ writeComposeFileInternal(t, filepath.Join(root, "group", "nested"))++ discovered, err := DiscoverProjectDirectories(root, false, 0)+ require.NoError(t, err)++ names := discoveredNamesInternal(discovered)+ require.Equal(t, []string{"nested"}, names)+}diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.goindex ee9db8627..1bcb6ef8f 100644--- a/backend/pkg/scheduler/filesystem_watcher_job.go+++ b/backend/pkg/scheduler/filesystem_watcher_job.go@@ -16,6 +16,7 @@ type FilesystemWatcherJob struct {projectService *services.ProjectServicetemplateService *services.TemplateServicesettingsService *services.SettingsService+ projectScanDepth intprojectsWatcher *fswatch.WatchertemplatesWatcher *fswatch.Watchermu sync.Mutex@@ -25,16 +26,18 @@ func NewFilesystemWatcherJob(projectService *services.ProjectService,templateService *services.TemplateService,settingsService *services.SettingsService,+ projectScanDepth int,) *FilesystemWatcherJob {return &FilesystemWatcherJob{- projectService: projectService,- templateService: templateService,- settingsService: settingsService,+ projectService: projectService,+ templateService: templateService,+ settingsService: settingsService,+ projectScanDepth: projectScanDepth,}}-func RegisterFilesystemWatcherJob(ctx context.Context, projectService *services.ProjectService, templateService *services.TemplateService, settingsService *services.SettingsService) (*FilesystemWatcherJob, error) {- job := NewFilesystemWatcherJob(projectService, templateService, settingsService)+func RegisterFilesystemWatcherJob(ctx context.Context, projectService *services.ProjectService, templateService *services.TemplateService, settingsService *services.SettingsService, projectScanDepth int) (*FilesystemWatcherJob, error) {+ job := NewFilesystemWatcherJob(projectService, templateService, settingsService, projectScanDepth)go func() {if err := job.Start(ctx); err != nil {@@ -58,12 +61,7 @@ func (j *FilesystemWatcherJob) Start(ctx context.Context) error {followProjectSymlinks := settings.FollowProjectSymlinks.IsTrue()j.logRecursiveProjectsWatchLimitWarningInternal(ctx, projectsDirectory)- sw, err := fswatch.NewWatcher(projectsDirectory, fswatch.WatcherOptions{- Debounce: 3 * time.Second, // Wait 3 seconds after last change before syncing- OnChange: j.handleFilesystemChange,- MaxDepth: 0,- FollowSymlinkDirs: followProjectSymlinks,- })+ sw, err := fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(followProjectSymlinks))if err != nil {return err}@@ -193,12 +191,7 @@ func (j *FilesystemWatcherJob) RestartProjectsWatcher(ctx context.Context) errorj.logRecursiveProjectsWatchLimitWarningInternal(ctx, projectsDirectory)// Create a new watcher with the updated path- sw, err := fswatch.NewWatcher(projectsDirectory, fswatch.WatcherOptions{- Debounce: 3 * time.Second,- OnChange: j.handleFilesystemChange,- MaxDepth: 0,- FollowSymlinkDirs: followProjectSymlinks,- })+ sw, err := fswatch.NewWatcher(projectsDirectory, j.projectWatcherOptionsInternal(followProjectSymlinks))if err != nil {return err}@@ -232,3 +225,12 @@ func (j *FilesystemWatcherJob) logRecursiveProjectsWatchLimitWarningInternal(ctx"path", projectsDirectory,"sysctl", "fs.inotify.max_user_watches")}++func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlinks bool) fswatch.WatcherOptions {+ return fswatch.WatcherOptions{+ Debounce: 3 * time.Second, // Wait 3 seconds after last change before syncing+ OnChange: j.handleFilesystemChange,+ MaxDepth: j.projectScanDepth,+ FollowSymlinkDirs: followProjectSymlinks,+ }+}diff --git a/backend/pkg/scheduler/filesystem_watcher_job_test.go b/backend/pkg/scheduler/filesystem_watcher_job_test.gonew file mode 100644index 000000000..e6a8a37cc--- /dev/null+++ b/backend/pkg/scheduler/filesystem_watcher_job_test.go@@ -0,0 +1,18 @@+package scheduler++import (+ "testing"++ "github.com/stretchr/testify/assert"+)++func TestFilesystemWatcherJob_ProjectWatcherOptions_UsesConfiguredMaxDepth(t *testing.T) {+ job := &FilesystemWatcherJob{+ projectScanDepth: 1,+ }++ opts := job.projectWatcherOptionsInternal(true)++ assert.Equal(t, 1, opts.MaxDepth)+ assert.True(t, opts.FollowSymlinkDirs)+}