Skip to content

Modern Docker Management, Designed for Everyone

fix: project sync depth not respected during FS sync (#2833)

Kyle Mendell09737ceaf5a5
diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.yml
deleted file mode 100644
index cf12349b3..000000000
--- a/.github/workflows/hadolint.yml
+++ /dev/null
@@ -1,58 +0,0 @@
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-# hadoint is a Dockerfile linter written in Haskell
-# that helps you build best practice Docker images.
-# More details at https://github.com/hadolint/hadolint
-
-name: Hadolint
-
-on:
- push:
- branches: ["main"]
- pull_request:
- # The branches below must be a subset of the branches above
- branches: ["main"]
- schedule:
- - cron: "24 19 * * 2"
-
-permissions:
- contents: read
-
-jobs:
- hadolint:
- name: Run hadolint scanning
- runs-on: depot-ubuntu-latest
- permissions:
- contents: read # for actions/checkout to fetch code
- security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
- actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
- strategy:
- fail-fast: false
- matrix:
- include:
- - name: manager
- dockerfile: ./docker/Dockerfile
- output: hadolint-results-manager.sarif
- - name: agent
- dockerfile: ./docker/Dockerfile-agent
- output: hadolint-results-agent.sarif
- steps:
- - name: Checkout code
- uses: actions/checkout@v6
-
- - name: Run hadolint
- uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5
- with:
- dockerfile: ${{ matrix.dockerfile }}
- format: sarif
- output-file: ${{ matrix.output }}
- no-fail: true
-
- - name: Upload analysis results to GitHub
- uses: github/codeql-action/[email protected]
- with:
- sarif_file: ${{ matrix.output }}
- category: hadolint-${{ matrix.name }}
- wait-for-processing: true
diff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.go
index 3d57e327d..3ebedaf77 100644
--- a/backend/internal/services/project_service.go
+++ b/backend/internal/services/project_service.go
@@ -1528,7 +1528,7 @@ func (s *ProjectService) SyncProjectsFromFileSystem(ctx context.Context) error {
seen[discoveredProject.Path] = struct{}{}
}
- if cerr := s.cleanupDBProjects(ctx, seen, followProjectSymlinks); cerr != nil {
+ if cerr := s.cleanupDBProjectsInternal(ctx, seen, followProjectSymlinks, projectsDir, s.config.ProjectScanMaxDepth); cerr != nil {
slog.WarnContext(ctx, "error during DB cleanup of projects", "error", cerr)
}
@@ -1601,54 +1601,92 @@ func (s *ProjectService) upsertProjectForDir(ctx context.Context, dirName, dirPa
return nil
}
-func (s *ProjectService) cleanupDBProjects(ctx context.Context, seen map[string]struct{}, followProjectSymlinks bool) error {
+func (s *ProjectService) cleanupDBProjectsInternal(ctx context.Context, seen map[string]struct{}, followProjectSymlinks bool, projectsDir string, maxDepth int) error {
var all []models.Project
if err := s.db.WithContext(ctx).Find(&all).Error; err != nil {
return fmt.Errorf("list projects for cleanup failed: %w", err)
}
for _, p := range all {
- // Skip paths seen in this pass
- if _, ok := seen[p.Path]; ok {
+ if skipProjectCleanupInternal(p, seen) {
continue
}
+ s.cleanupUnseenProjectInternal(ctx, p, followProjectSymlinks, projectsDir, maxDepth)
+ }
+ return nil
+}
- // Skip projects whose lifecycle is owned by the gitops system.
- // Their compose files may not exist on disk yet (e.g. during a sync
- // or after an SSH/clone failure) and should never be deleted here.
- if p.GitOpsManagedBy != nil && strings.TrimSpace(*p.GitOpsManagedBy) != "" {
- continue
- }
+func skipProjectCleanupInternal(p models.Project, seen map[string]struct{}) bool {
+ // Skip paths seen in this pass.
+ if _, ok := seen[p.Path]; ok {
+ return true
+ }
- validDir, err := projects.IsProjectDirectoryPath(p.Path, followProjectSymlinks)
- if err != nil {
- if os.IsNotExist(err) {
- if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {
- slog.WarnContext(ctx, "failed to delete missing-path project", "projectID", p.ID, "error", derr)
- }
- continue
- }
- slog.WarnContext(ctx, "stat error during cleanup", "path", p.Path, "error", err)
- continue
- }
- if !validDir {
- if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {
- slog.WarnContext(ctx, "failed to delete non-project path", "projectID", p.ID, "path", p.Path, "error", derr)
- }
- continue
- }
+ // Skip projects whose lifecycle is owned by the gitops system. Their compose
+ // files may not exist on disk yet (e.g. during a sync or after an SSH/clone
+ // failure) and should never be deleted here.
+ return p.GitOpsManagedBy != nil && strings.TrimSpace(*p.GitOpsManagedBy) != ""
+}
- if _, err := s.resolveProjectComposeFileInternal(ctx, &p); err != nil {
- if _, ok := errors.AsType[*common.ProjectComposeFileNotFoundError](err); !ok {
- slog.WarnContext(ctx, "failed to validate project compose file during cleanup", "projectID", p.ID, "path", p.Path, "error", err)
- continue
- }
- if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {
- slog.WarnContext(ctx, "failed to delete project without compose", "projectID", p.ID, "error", derr)
- }
+func (s *ProjectService) cleanupUnseenProjectInternal(ctx context.Context, p models.Project, followProjectSymlinks bool, projectsDir string, maxDepth int) {
+ if s.projectExceedsScanDepthInternal(p, projectsDir, maxDepth) {
+ s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete project beyond configured scan depth", "path", p.Path)
+ return
+ }
+
+ validDir, err := projects.IsProjectDirectoryPath(p.Path, followProjectSymlinks)
+ if err != nil {
+ s.cleanupProjectPathErrorInternal(ctx, p, err)
+ return
+ }
+ if !validDir {
+ s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete non-project path", "path", p.Path)
+ return
+ }
+
+ s.cleanupProjectComposeFileInternal(ctx, p)
+}
+
+func (s *ProjectService) projectExceedsScanDepthInternal(p models.Project, projectsDir string, maxDepth int) bool {
+ // Remove projects that still exist on disk but now fall outside the configured
+ // scan depth (e.g. after PROJECT_SCAN_MAX_DEPTH was lowered). They are no
+ // longer discovered, so they must not linger in the list. Projects at the
+ // projects root or outside it (relativePath == "") are left to the on-disk
+ // validation below.
+ if maxDepth <= 0 {
+ return false
+ }
+
+ rel := s.getProjectRelativePathInternal(projectsDir, p.Path)
+ return rel != "" && strings.Count(rel, "/")+1 > maxDepth
+}
+
+func (s *ProjectService) cleanupProjectPathErrorInternal(ctx context.Context, p models.Project, err error) {
+ if os.IsNotExist(err) {
+ s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete missing-path project")
+ return
+ }
+
+ slog.WarnContext(ctx, "stat error during cleanup", "path", p.Path, "error", err)
+}
+
+func (s *ProjectService) cleanupProjectComposeFileInternal(ctx context.Context, p models.Project) {
+ if _, err := s.resolveProjectComposeFileInternal(ctx, &p); err != nil {
+ if _, ok := errors.AsType[*common.ProjectComposeFileNotFoundError](err); !ok {
+ slog.WarnContext(ctx, "failed to validate project compose file during cleanup", "projectID", p.ID, "path", p.Path, "error", err)
+ return
}
+ s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete project without compose")
+ }
+}
+
+func (s *ProjectService) deleteProjectDuringCleanupInternal(ctx context.Context, p models.Project, failureMessage string, attrs ...any) {
+ if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {
+ logAttrs := make([]any, 0, 4+len(attrs))
+ logAttrs = append(logAttrs, "projectID", p.ID, "error", derr)
+ logAttrs = append(logAttrs, attrs...)
+ slog.WarnContext(ctx, failureMessage, logAttrs...)
}
- return nil
}
func (s *ProjectService) ListAllProjects(ctx context.Context) ([]models.Project, error) {
diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.go
index 9a6bac937..21a71904b 100644
--- a/backend/internal/services/project_service_test.go
+++ b/backend/internal/services/project_service_test.go
@@ -3606,6 +3606,45 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesDeletedNestedProject(t
assert.Empty(t, items)
}
+func TestProjectService_SyncProjectsFromFileSystem_RemovesProjectsBeyondReducedScanMaxDepth(t *testing.T) {
+ db := setupProjectTestDB(t)
+ ctx := context.Background()
+
+ settingsService, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ projectsRoot := t.TempDir()
+ topLevelPath := createComposeProjectDir(t, projectsRoot, "project1")
+ nestedPath := createComposeProjectDir(t, projectsRoot, filepath.Join("group", "project2"))
+
+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
+
+ // Initial sync at the default scan depth discovers both the top-level and
+ // the nested project, persisting them to the database.
+ defaultSvc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())
+ require.NoError(t, defaultSvc.SyncProjectsFromFileSystem(ctx))
+
+ items, err := defaultSvc.ListAllProjects(ctx)
+ require.NoError(t, err)
+ require.Len(t, items, 2)
+
+ // Lowering the scan depth must prune the nested project from the database on
+ // the next sync, even though its compose file still exists on disk.
+ t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")
+ depthLimitedSvc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())
+ require.NoError(t, depthLimitedSvc.SyncProjectsFromFileSystem(ctx))
+
+ items, err = depthLimitedSvc.ListAllProjects(ctx)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ assert.Equal(t, "project1", items[0].Name)
+ assert.Equal(t, topLevelPath, items[0].Path)
+
+ // The pruned project's files must remain untouched on disk so raising the
+ // depth again re-discovers it.
+ assert.FileExists(t, filepath.Join(nestedPath, "compose.yaml"))
+}
+
func TestProjectService_SyncProjectsFromFileSystem_PreservesDBRecordsWhenDirectoryUnreadable(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("POSIX permission-denied behavior is not portable to Windows")
diff --git a/tests/spec/project.spec.ts b/tests/spec/project.spec.ts
index 76e39d993..068bb1f37 100644
--- a/tests/spec/project.spec.ts
+++ b/tests/spec/project.spec.ts
@@ -676,6 +676,8 @@ test.describe('New Compose Project Page', () => {
.toBe('running');
expect(projectPullRequestCount).toBe(0);
+ await page.reload();
+ await page.waitForLoadState('load');
await expect(page.getByText('Running', { exact: true }).first()).toBeVisible({
timeout: 20000
});