Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: project sync depth not respected during FS sync (#2833)
Kyle Mendell09737ceaf5a5
diff --git a/.github/workflows/hadolint.yml b/.github/workflows/hadolint.ymldeleted file mode 100644index cf12349b3..000000000--- a/.github/workflows/hadolint.yml+++ /dev/null@@ -1,58 +0,0 @@-# This workflow uses actions that are not certified by GitHub.-# They are provided by a third-party and are governed by-# separate terms of service, privacy policy, and support-# documentation.-# hadoint is a Dockerfile linter written in Haskell-# that helps you build best practice Docker images.-# More details at https://github.com/hadolint/hadolint--name: Hadolint--on:- push:- branches: ["main"]- pull_request:- # The branches below must be a subset of the branches above- branches: ["main"]- schedule:- - cron: "24 19 * * 2"--permissions:- contents: read--jobs:- hadolint:- name: Run hadolint scanning- runs-on: depot-ubuntu-latest- permissions:- contents: read # for actions/checkout to fetch code- security-events: write # for github/codeql-action/upload-sarif to upload SARIF results- actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status- strategy:- fail-fast: false- matrix:- include:- - name: manager- dockerfile: ./docker/Dockerfile- output: hadolint-results-manager.sarif- - name: agent- dockerfile: ./docker/Dockerfile-agent- output: hadolint-results-agent.sarif- steps:- - name: Checkout code- uses: actions/checkout@v6-- - name: Run hadolint- uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5- with:- dockerfile: ${{ matrix.dockerfile }}- format: sarif- output-file: ${{ matrix.output }}- no-fail: true-- - name: Upload analysis results to GitHub- uses: github/codeql-action/[email protected]- with:- sarif_file: ${{ matrix.output }}- category: hadolint-${{ matrix.name }}- wait-for-processing: truediff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.goindex 3d57e327d..3ebedaf77 100644--- a/backend/internal/services/project_service.go+++ b/backend/internal/services/project_service.go@@ -1528,7 +1528,7 @@ func (s *ProjectService) SyncProjectsFromFileSystem(ctx context.Context) error {seen[discoveredProject.Path] = struct{}{}}- if cerr := s.cleanupDBProjects(ctx, seen, followProjectSymlinks); cerr != nil {+ if cerr := s.cleanupDBProjectsInternal(ctx, seen, followProjectSymlinks, projectsDir, s.config.ProjectScanMaxDepth); cerr != nil {slog.WarnContext(ctx, "error during DB cleanup of projects", "error", cerr)}@@ -1601,54 +1601,92 @@ func (s *ProjectService) upsertProjectForDir(ctx context.Context, dirName, dirPareturn nil}-func (s *ProjectService) cleanupDBProjects(ctx context.Context, seen map[string]struct{}, followProjectSymlinks bool) error {+func (s *ProjectService) cleanupDBProjectsInternal(ctx context.Context, seen map[string]struct{}, followProjectSymlinks bool, projectsDir string, maxDepth int) error {var all []models.Projectif err := s.db.WithContext(ctx).Find(&all).Error; err != nil {return fmt.Errorf("list projects for cleanup failed: %w", err)}for _, p := range all {- // Skip paths seen in this pass- if _, ok := seen[p.Path]; ok {+ if skipProjectCleanupInternal(p, seen) {continue}+ s.cleanupUnseenProjectInternal(ctx, p, followProjectSymlinks, projectsDir, maxDepth)+ }+ return nil+}- // Skip projects whose lifecycle is owned by the gitops system.- // Their compose files may not exist on disk yet (e.g. during a sync- // or after an SSH/clone failure) and should never be deleted here.- if p.GitOpsManagedBy != nil && strings.TrimSpace(*p.GitOpsManagedBy) != "" {- continue- }+func skipProjectCleanupInternal(p models.Project, seen map[string]struct{}) bool {+ // Skip paths seen in this pass.+ if _, ok := seen[p.Path]; ok {+ return true+ }- validDir, err := projects.IsProjectDirectoryPath(p.Path, followProjectSymlinks)- if err != nil {- if os.IsNotExist(err) {- if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {- slog.WarnContext(ctx, "failed to delete missing-path project", "projectID", p.ID, "error", derr)- }- continue- }- slog.WarnContext(ctx, "stat error during cleanup", "path", p.Path, "error", err)- continue- }- if !validDir {- if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {- slog.WarnContext(ctx, "failed to delete non-project path", "projectID", p.ID, "path", p.Path, "error", derr)- }- continue- }+ // Skip projects whose lifecycle is owned by the gitops system. Their compose+ // files may not exist on disk yet (e.g. during a sync or after an SSH/clone+ // failure) and should never be deleted here.+ return p.GitOpsManagedBy != nil && strings.TrimSpace(*p.GitOpsManagedBy) != ""+}- if _, err := s.resolveProjectComposeFileInternal(ctx, &p); err != nil {- if _, ok := errors.AsType[*common.ProjectComposeFileNotFoundError](err); !ok {- slog.WarnContext(ctx, "failed to validate project compose file during cleanup", "projectID", p.ID, "path", p.Path, "error", err)- continue- }- if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {- slog.WarnContext(ctx, "failed to delete project without compose", "projectID", p.ID, "error", derr)- }+func (s *ProjectService) cleanupUnseenProjectInternal(ctx context.Context, p models.Project, followProjectSymlinks bool, projectsDir string, maxDepth int) {+ if s.projectExceedsScanDepthInternal(p, projectsDir, maxDepth) {+ s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete project beyond configured scan depth", "path", p.Path)+ return+ }++ validDir, err := projects.IsProjectDirectoryPath(p.Path, followProjectSymlinks)+ if err != nil {+ s.cleanupProjectPathErrorInternal(ctx, p, err)+ return+ }+ if !validDir {+ s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete non-project path", "path", p.Path)+ return+ }++ s.cleanupProjectComposeFileInternal(ctx, p)+}++func (s *ProjectService) projectExceedsScanDepthInternal(p models.Project, projectsDir string, maxDepth int) bool {+ // Remove projects that still exist on disk but now fall outside the configured+ // scan depth (e.g. after PROJECT_SCAN_MAX_DEPTH was lowered). They are no+ // longer discovered, so they must not linger in the list. Projects at the+ // projects root or outside it (relativePath == "") are left to the on-disk+ // validation below.+ if maxDepth <= 0 {+ return false+ }++ rel := s.getProjectRelativePathInternal(projectsDir, p.Path)+ return rel != "" && strings.Count(rel, "/")+1 > maxDepth+}++func (s *ProjectService) cleanupProjectPathErrorInternal(ctx context.Context, p models.Project, err error) {+ if os.IsNotExist(err) {+ s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete missing-path project")+ return+ }++ slog.WarnContext(ctx, "stat error during cleanup", "path", p.Path, "error", err)+}++func (s *ProjectService) cleanupProjectComposeFileInternal(ctx context.Context, p models.Project) {+ if _, err := s.resolveProjectComposeFileInternal(ctx, &p); err != nil {+ if _, ok := errors.AsType[*common.ProjectComposeFileNotFoundError](err); !ok {+ slog.WarnContext(ctx, "failed to validate project compose file during cleanup", "projectID", p.ID, "path", p.Path, "error", err)+ return}+ s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete project without compose")+ }+}++func (s *ProjectService) deleteProjectDuringCleanupInternal(ctx context.Context, p models.Project, failureMessage string, attrs ...any) {+ if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {+ logAttrs := make([]any, 0, 4+len(attrs))+ logAttrs = append(logAttrs, "projectID", p.ID, "error", derr)+ logAttrs = append(logAttrs, attrs...)+ slog.WarnContext(ctx, failureMessage, logAttrs...)}- return nil}func (s *ProjectService) ListAllProjects(ctx context.Context) ([]models.Project, error) {diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.goindex 9a6bac937..21a71904b 100644--- a/backend/internal/services/project_service_test.go+++ b/backend/internal/services/project_service_test.go@@ -3606,6 +3606,45 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesDeletedNestedProject(tassert.Empty(t, items)}+func TestProjectService_SyncProjectsFromFileSystem_RemovesProjectsBeyondReducedScanMaxDepth(t *testing.T) {+ db := setupProjectTestDB(t)+ ctx := context.Background()++ settingsService, err := NewSettingsService(ctx, db)+ require.NoError(t, err)++ projectsRoot := t.TempDir()+ topLevelPath := createComposeProjectDir(t, projectsRoot, "project1")+ nestedPath := createComposeProjectDir(t, projectsRoot, filepath.Join("group", "project2"))++ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))++ // Initial sync at the default scan depth discovers both the top-level and+ // the nested project, persisting them to the database.+ defaultSvc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())+ require.NoError(t, defaultSvc.SyncProjectsFromFileSystem(ctx))++ items, err := defaultSvc.ListAllProjects(ctx)+ require.NoError(t, err)+ require.Len(t, items, 2)++ // Lowering the scan depth must prune the nested project from the database on+ // the next sync, even though its compose file still exists on disk.+ t.Setenv("PROJECT_SCAN_MAX_DEPTH", "1")+ depthLimitedSvc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())+ require.NoError(t, depthLimitedSvc.SyncProjectsFromFileSystem(ctx))++ items, err = depthLimitedSvc.ListAllProjects(ctx)+ require.NoError(t, err)+ require.Len(t, items, 1)+ assert.Equal(t, "project1", items[0].Name)+ assert.Equal(t, topLevelPath, items[0].Path)++ // The pruned project's files must remain untouched on disk so raising the+ // depth again re-discovers it.+ assert.FileExists(t, filepath.Join(nestedPath, "compose.yaml"))+}+func TestProjectService_SyncProjectsFromFileSystem_PreservesDBRecordsWhenDirectoryUnreadable(t *testing.T) {if runtime.GOOS == "windows" {t.Skip("POSIX permission-denied behavior is not portable to Windows")diff --git a/tests/spec/project.spec.ts b/tests/spec/project.spec.tsindex 76e39d993..068bb1f37 100644--- a/tests/spec/project.spec.ts+++ b/tests/spec/project.spec.ts@@ -676,6 +676,8 @@ test.describe('New Compose Project Page', () => {.toBe('running');expect(projectPullRequestCount).toBe(0);+ await page.reload();+ await page.waitForLoadState('load');await expect(page.getByText('Running', { exact: true }).first()).toBeVisible({timeout: 20000});