Skip to content

Modern Docker Management, Designed for Everyone

fix: projects disapearing after fs sync (#2940)

Kyle Mendell42a861d02f5e
diff --git a/backend/internal/common/errors.go b/backend/internal/common/errors.go
index 3923e1d04..64dd048aa 100644
--- a/backend/internal/common/errors.go
+++ b/backend/internal/common/errors.go
@@ -852,6 +852,32 @@ func (e *ProjectComposeFileNotFoundError) Error() string {
return fmt.Sprintf("Project compose file not found: %v", e.Err)
}
+func (e *ProjectComposeFileNotFoundError) Unwrap() error {
+ return e.Err
+}
+
+// ComposeFileNotFoundError indicates a directory contains no recognizable
+// compose file at all (zero candidates).
+type ComposeFileNotFoundError struct {
+ Dir string
+}
+
+func (e *ComposeFileNotFoundError) Error() string {
+ return fmt.Sprintf("no compose file found in %q", e.Dir)
+}
+
+// AmbiguousComposeFileError indicates a directory contains multiple custom-named
+// compose candidates that cannot be disambiguated. The directory still holds
+// compose content, so callers must not treat this as "no project here" (e.g. by
+// deleting the project record).
+type AmbiguousComposeFileError struct {
+ Dir string
+}
+
+func (e *AmbiguousComposeFileError) Error() string {
+ return fmt.Sprintf("multiple custom compose files found in %q", e.Dir)
+}
+
type ProjectDiscoveryError struct {
Dir string
Err error
diff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.go
index 07e6e7049..61e3f2857 100644
--- a/backend/internal/services/project_service.go
+++ b/backend/internal/services/project_service.go
@@ -1631,21 +1631,67 @@ func (s *ProjectService) upsertProjectForDir(ctx context.Context, dirName, dirPa
return nil
}
+// projectCleanupDecision records a project the reconcile pass intends to delete,
+// alongside the reason logged when the deletion is carried out.
+type projectCleanupDecision struct {
+ project models.Project
+ reason string
+}
+
func (s *ProjectService) cleanupDBProjectsInternal(ctx context.Context, seen map[string]struct{}, followProjectSymlinks bool, projectsDir string, maxDepth int) error {
var all []models.Project
if err := s.db.WithContext(ctx).Find(&all).Error; err != nil {
return fmt.Errorf("list projects for cleanup failed: %w", err)
}
+ // Decide deletions without performing them. Collecting decisions up front lets
+ // the mass-wipe guard veto an entire suspicious pass (e.g. the projects volume
+ // is unmounted, so every path is missing at once) before any rows are removed.
+ candidates := 0
+ pendingDeletions := make([]projectCleanupDecision, 0)
for _, p := range all {
if skipProjectCleanupInternal(p, seen) {
continue
}
- s.cleanupUnseenProjectInternal(ctx, p, followProjectSymlinks, projectsDir, maxDepth)
+ candidates++
+ if decision, remove := s.evaluateProjectCleanupInternal(ctx, p, followProjectSymlinks, projectsDir, maxDepth); remove {
+ pendingDeletions = append(pendingDeletions, decision)
+ }
+ }
+
+ if s.cleanupWouldMassWipeInternal(ctx, candidates, len(pendingDeletions), projectsDir) {
+ return nil
+ }
+
+ for _, decision := range pendingDeletions {
+ s.deleteProjectDuringCleanupInternal(ctx, decision.project, decision.reason)
}
return nil
}
+// cleanupWouldMassWipeInternal reports whether the pending deletions look like an
+// accidental mass wipe rather than legitimate removals. It engages when a single
+// pass would prune more than one project AND more than half of the cleanup
+// candidates — so the table cannot be near-emptied at once (e.g. when the projects
+// directory is unmounted or mis-mapped and every path goes missing), no matter how
+// few projects the deployment has. A single removal is always allowed: it is
+// indistinguishable from a legitimate "deleted my only project" and is not a mass
+// wipe. When the guard engages it logs a WARN pointing the operator at the likely
+// volume/mount misconfiguration and the caller skips every deletion in the pass.
+func (s *ProjectService) cleanupWouldMassWipeInternal(ctx context.Context, candidates, deleteCount int, projectsDir string) bool {
+ if deleteCount <= 1 || deleteCount*2 <= candidates {
+ return false
+ }
+
+ slog.WarnContext(ctx,
+ "skipping project cleanup: this reconcile would delete most projects in a single pass, which usually means the projects directory is empty, unmounted, or mis-mapped; preserving DB records — check the projects volume is mounted and mapped correctly",
+ "wouldDelete", deleteCount,
+ "cleanupCandidates", candidates,
+ "projectsDir", projectsDir,
+ )
+ return true
+}
+
func skipProjectCleanupInternal(p models.Project, seen map[string]struct{}) bool {
// Skip paths seen in this pass.
if _, ok := seen[p.Path]; ok {
@@ -1658,23 +1704,24 @@ func skipProjectCleanupInternal(p models.Project, seen map[string]struct{}) bool
return p.GitOpsManagedBy != nil && strings.TrimSpace(*p.GitOpsManagedBy) != ""
}
-func (s *ProjectService) cleanupUnseenProjectInternal(ctx context.Context, p models.Project, followProjectSymlinks bool, projectsDir string, maxDepth int) {
+// evaluateProjectCleanupInternal decides whether a project that was not seen in
+// the current filesystem pass should be pruned. It performs only read-only checks
+// (warning in place for the "keep" cases); the actual deletion is deferred to the
+// caller so the mass-wipe guard can veto an entire suspicious pass.
+func (s *ProjectService) evaluateProjectCleanupInternal(ctx context.Context, p models.Project, followProjectSymlinks bool, projectsDir string, maxDepth int) (projectCleanupDecision, bool) {
if s.projectExceedsScanDepthInternal(p, projectsDir, maxDepth) {
- s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete project beyond configured scan depth", "path", p.Path)
- return
+ return projectCleanupDecision{project: p, reason: "removed project: directory is beyond the configured scan depth"}, true
}
validDir, err := projects.IsProjectDirectoryPath(p.Path, followProjectSymlinks)
if err != nil {
- s.cleanupProjectPathErrorInternal(ctx, p, err)
- return
+ return s.evaluateProjectPathErrorInternal(ctx, p, err)
}
if !validDir {
- s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete non-project path", "path", p.Path)
- return
+ return projectCleanupDecision{project: p, reason: "removed project: path is no longer a valid project directory"}, true
}
- s.cleanupProjectComposeFileInternal(ctx, p)
+ return s.evaluateProjectComposeFileInternal(ctx, p)
}
func (s *ProjectService) projectExceedsScanDepthInternal(p models.Project, projectsDir string, maxDepth int) bool {
@@ -1691,32 +1738,53 @@ func (s *ProjectService) projectExceedsScanDepthInternal(p models.Project, proje
return rel != "" && strings.Count(rel, "/")+1 > maxDepth
}
-func (s *ProjectService) cleanupProjectPathErrorInternal(ctx context.Context, p models.Project, err error) {
+func (s *ProjectService) evaluateProjectPathErrorInternal(ctx context.Context, p models.Project, err error) (projectCleanupDecision, bool) {
if os.IsNotExist(err) {
- s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete missing-path project")
- return
+ return projectCleanupDecision{project: p, reason: "removed project: directory no longer exists"}, true
}
- slog.WarnContext(ctx, "stat error during cleanup", "path", p.Path, "error", err)
+ slog.WarnContext(ctx, "stat error during cleanup; keeping DB record", "path", p.Path, "error", err)
+ return projectCleanupDecision{}, false
}
-func (s *ProjectService) cleanupProjectComposeFileInternal(ctx context.Context, p models.Project) {
- if _, err := s.resolveProjectComposeFileInternal(ctx, &p); err != nil {
- if _, ok := errors.AsType[*common.ProjectComposeFileNotFoundError](err); !ok {
- slog.WarnContext(ctx, "failed to validate project compose file during cleanup", "projectID", p.ID, "path", p.Path, "error", err)
- return
- }
- s.deleteProjectDuringCleanupInternal(ctx, p, "failed to delete project without compose")
+func (s *ProjectService) evaluateProjectComposeFileInternal(ctx context.Context, p models.Project) (projectCleanupDecision, bool) {
+ _, err := s.resolveProjectComposeFileInternal(ctx, &p)
+ if err == nil {
+ return projectCleanupDecision{}, false
}
+
+ // The project directory still exists here (it passed the directory-validity
+ // check above). Only prune the DB record when the directory genuinely has no
+ // compose file. Any other resolution failure — an ambiguous match ("multiple
+ // custom compose files"), an unreadable directory, a transient parse error —
+ // means the project still has compose content on disk and may be deployable.
+ // Deleting it would silently destroy a live project whose files are intact, so
+ // keep the record and warn instead.
+ if _, ok := errors.AsType[*common.ComposeFileNotFoundError](err); !ok {
+ slog.WarnContext(ctx, "project directory present but compose file unresolved during cleanup; keeping DB record",
+ "projectID", p.ID, "path", p.Path, "error", err)
+ return projectCleanupDecision{}, false
+ }
+
+ return projectCleanupDecision{project: p, reason: "removed orphaned project: directory present but contains no compose file"}, true
}
-func (s *ProjectService) deleteProjectDuringCleanupInternal(ctx context.Context, p models.Project, failureMessage string, attrs ...any) {
+// deleteProjectDuringCleanupInternal removes a project record discovered to be
+// stale during the filesystem reconcile. Every removal is logged (WARN on
+// success, ERROR on failure) so this destructive operation always leaves an
+// audit trail — previously successful deletions were silent.
+func (s *ProjectService) deleteProjectDuringCleanupInternal(ctx context.Context, p models.Project, reason string, attrs ...any) {
+ logAttrs := make([]any, 0, 6+len(attrs))
+ logAttrs = append(logAttrs, "projectID", p.ID, "name", p.Name, "path", p.Path)
+ logAttrs = append(logAttrs, attrs...)
+
if derr := s.db.WithContext(ctx).Delete(&models.Project{}, "id = ?", p.ID).Error; derr != nil {
- logAttrs := make([]any, 0, 4+len(attrs))
- logAttrs = append(logAttrs, "projectID", p.ID, "error", derr)
- logAttrs = append(logAttrs, attrs...)
- slog.WarnContext(ctx, failureMessage, logAttrs...)
+ slog.ErrorContext(ctx, "failed to delete project during filesystem cleanup",
+ append(logAttrs, "reason", reason, "error", derr)...)
+ return
}
+
+ slog.WarnContext(ctx, reason, logAttrs...)
}
func (s *ProjectService) ListAllProjects(ctx context.Context) ([]models.Project, error) {
diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.go
index cd90b2af2..ea116b2d9 100644
--- a/backend/internal/services/project_service_test.go
+++ b/backend/internal/services/project_service_test.go
@@ -3849,6 +3849,77 @@ func TestProjectService_SyncProjectsFromFileSystem_RemovesDeletedNestedProject(t
assert.Empty(t, items)
}
+func TestProjectService_SyncProjectsFromFileSystem_PreservesProjectsWhenDirectoryEmptyOrUnmounted(t *testing.T) {
+ db := setupProjectTestDB(t)
+ ctx := context.Background()
+
+ settingsService, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ // An existing-but-empty projects directory simulates a mis-mapped or unmounted
+ // projects volume: GetProjectsDirectory resolves (and MkdirAll's) it, discovery
+ // finds nothing, and every stored project path is now missing on disk.
+ projectsRoot := t.TempDir()
+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
+
+ // Seed a small deployment's worth of filesystem-managed projects whose
+ // directories do not exist. Each would individually be pruned as "directory no
+ // longer exists"; together they would wipe the table — exactly what the
+ // mass-wipe guard must prevent, even for deployments with only a handful of
+ // projects (the guard must not give small installs a free pass).
+ const seeded = 3
+ for i := range seeded {
+ require.NoError(t, db.WithContext(ctx).Create(&models.Project{
+ Name: fmt.Sprintf("project-%d", i),
+ Path: filepath.Join(projectsRoot, fmt.Sprintf("project-%d", i)),
+ Status: models.ProjectStatusStopped,
+ }).Error)
+ }
+
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())
+ require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
+
+ items, err := svc.ListAllProjects(ctx)
+ require.NoError(t, err)
+ assert.Len(t, items, seeded, "an empty/mis-mapped projects directory must not wipe existing project records")
+}
+
+func TestProjectService_SyncProjectsFromFileSystem_PreservesProjectWithAmbiguousCustomCompose(t *testing.T) {
+ db := setupProjectTestDB(t)
+ ctx := context.Background()
+
+ settingsService, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ projectsRoot := t.TempDir()
+ projectPath := createComposeProjectDir(t, projectsRoot, "ambiguous-project")
+
+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsRoot))
+
+ svc := NewProjectService(db, settingsService, nil, nil, nil, nil, config.Load())
+ require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
+
+ items, err := svc.ListAllProjects(ctx)
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+
+ // Replace the standard compose.yaml with two custom-named compose files. The
+ // directory still holds compose content, but DetectComposeFile can't pick one
+ // and returns common.AmbiguousComposeFileError. The reconcile must NOT delete the
+ // record: the project's files are intact on disk and it may be deployable.
+ require.NoError(t, os.Remove(filepath.Join(projectPath, "compose.yaml")))
+ composeBody := []byte("services:\n app:\n image: nginx:alpine\n")
+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "app.yaml"), composeBody, 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "extra.yaml"), composeBody, 0o644))
+
+ require.NoError(t, svc.SyncProjectsFromFileSystem(ctx))
+
+ items, err = svc.ListAllProjects(ctx)
+ require.NoError(t, err)
+ assert.Len(t, items, 1, "project with ambiguous compose files must not be deleted")
+ assert.Equal(t, projectPath, items[0].Path)
+}
+
func TestProjectService_SyncProjectsFromFileSystem_RemovesProjectsBeyondReducedScanMaxDepth(t *testing.T) {
db := setupProjectTestDB(t)
ctx := context.Background()
diff --git a/backend/pkg/projects/fs_templates.go b/backend/pkg/projects/fs_templates.go
index c3f065c80..c060d0740 100644
--- a/backend/pkg/projects/fs_templates.go
+++ b/backend/pkg/projects/fs_templates.go
@@ -9,6 +9,7 @@ import (
"regexp"
"strings"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
pkgutils "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)
@@ -16,7 +17,7 @@ func ReadFolderComposeTemplate(baseDir, folder string) (string, *string, string,
folderPath := filepath.Join(baseDir, folder)
composePath, err := DetectComposeFile(folderPath)
if err != nil {
- if errors.Is(err, errComposeFileNotFoundInternal) {
+ if _, ok := errors.AsType[*common.ComposeFileNotFoundError](err); ok {
return "", nil, "", false, nil
}
return "", nil, "", false, fmt.Errorf("detect compose file: %w", err)
diff --git a/backend/pkg/projects/load.go b/backend/pkg/projects/load.go
index 6fbeae734..87827397e 100644
--- a/backend/pkg/projects/load.go
+++ b/backend/pkg/projects/load.go
@@ -2,7 +2,6 @@ package projects
import (
"context"
- "errors"
"fmt"
"log/slog"
"maps"
@@ -20,10 +19,9 @@ import (
composetypes "github.com/compose-spec/compose-go/v2/types"
"github.com/docker/compose/v5/pkg/api"
"github.com/docker/go-units"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
)
-var errComposeFileNotFoundInternal = errors.New("no compose file found")
-
var ProjectFileCandidates = []string{
"compose.yaml",
"compose.yml",
@@ -130,17 +128,17 @@ func DetectComposeFile(dir string) (string, error) {
case len(dirMatchedCandidates) == 1:
return dirMatchedCandidates[0], nil
case len(dirMatchedCandidates) > 1:
- return "", fmt.Errorf("multiple custom compose files found in %q", dir)
+ return "", &common.AmbiguousComposeFileError{Dir: dir}
case len(composeNamedCandidates) == 1:
return composeNamedCandidates[0], nil
case len(composeNamedCandidates) > 1:
- return "", fmt.Errorf("multiple custom compose files found in %q", dir)
+ return "", &common.AmbiguousComposeFileError{Dir: dir}
case len(customCandidates) == 1:
return customCandidates[0], nil
case len(customCandidates) > 1:
- return "", fmt.Errorf("multiple custom compose files found in %q", dir)
+ return "", &common.AmbiguousComposeFileError{Dir: dir}
default:
- return "", fmt.Errorf("%w in %q", errComposeFileNotFoundInternal, dir)
+ return "", &common.ComposeFileNotFoundError{Dir: dir}
}
}