Skip to content

Modern Docker Management, Designed for Everyone

fix: git sync is not pulling full folders, no longer deploy on creation of sync (#2239)

Kyle Mendell6ceefc826747
diff --git a/backend/internal/services/gitops_sync_service.go b/backend/internal/services/gitops_sync_service.go
index 2f3f03fa4..ab3043bd3 100644
--- a/backend/internal/services/gitops_sync_service.go
+++ b/backend/internal/services/gitops_sync_service.go
@@ -1,12 +1,14 @@
package services
import (
+ "bytes"
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"maps"
+ "os"
"path/filepath"
"strings"
"time"
@@ -46,6 +48,26 @@ type scheduledGitOpsSync struct {
LastSyncAt *time.Time
}
+// preparedSyncSource captures the repository data needed by the sync execution
+// paths after the source repository has been cloned and validated.
+type preparedSyncSource struct {
+ repoPath string
+ commitHash string
+ composeContent string
+ envContent *string
+}
+
+// stagedDirectorySync holds the fully prepared directory-sync result before it
+// is promoted into the live project path.
+type stagedDirectorySync struct {
+ stagePath string
+ composeFileName string
+ project *models.Project
+ syncedFiles []string
+ serviceCount int
+ contentsChanged bool
+}
+
func validateSyncLimits(maxFiles *int, maxTotalSize, maxBinarySize *int64) error {
if maxFiles != nil && *maxFiles < 0 {
return fmt.Errorf("maxSyncFiles must be non-negative")
@@ -458,123 +480,146 @@ func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id s
SyncedAt: time.Now(),
}
- // Get repository and auth config
+ source, err := s.prepareSyncSource(syncCtx, sync, result, actor)
+ if source != nil && source.repoPath != "" {
+ defer func() {
+ if cleanupErr := s.repoService.gitClient.Cleanup(source.repoPath); cleanupErr != nil {
+ slog.WarnContext(syncCtx, "Failed to cleanup repository", "path", source.repoPath, "error", cleanupErr)
+ }
+ }()
+ }
+ if err != nil {
+ return result, err
+ }
+
+ if sync.SyncDirectory {
+ return s.performDirectorySync(syncCtx, sync, id, actor, result, source)
+ }
+
+ return s.performSingleFileSync(syncCtx, sync, id, actor, result, source)
+}
+
+// prepareSyncSource clones the source repository, validates that the configured
+// compose file exists, and reads the compose/env inputs for the sync flow.
+func (s *GitOpsSyncService) prepareSyncSource(ctx context.Context, sync *models.GitOpsSync, result *gitops.SyncResult, actor models.User) (*preparedSyncSource, error) {
repository := sync.Repository
if repository == nil {
- return result, s.failSync(syncCtx, id, result, sync, actor, "Repository not found", "repository not found")
+ return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Repository not found", "repository not found")
}
- authConfig, err := s.repoService.GetAuthConfig(syncCtx, repository)
+ authConfig, err := s.repoService.GetAuthConfig(ctx, repository)
if err != nil {
- return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to get authentication config", err.Error())
+ return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to get authentication config", err.Error())
}
- // Clone the repository
- repoPath, err := s.repoService.gitClient.Clone(syncCtx, repository.URL, sync.Branch, authConfig)
+ repoPath, err := s.repoService.gitClient.Clone(ctx, repository.URL, sync.Branch, authConfig)
if err != nil {
- return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to clone repository", err.Error())
+ return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to clone repository", err.Error())
}
- defer func() {
- if cleanupErr := s.repoService.gitClient.Cleanup(repoPath); cleanupErr != nil {
- slog.WarnContext(syncCtx, "Failed to cleanup repository", "path", repoPath, "error", cleanupErr)
- }
- }()
- // Get the current commit hash
- commitHash, err := s.repoService.gitClient.GetCurrentCommit(syncCtx, repoPath)
+ commitHash, err := s.repoService.gitClient.GetCurrentCommit(ctx, repoPath)
if err != nil {
- slog.WarnContext(syncCtx, "Failed to get commit hash", "error", err)
+ slog.WarnContext(ctx, "Failed to get commit hash", "error", err)
commitHash = ""
}
- // Check if compose file exists
- if !s.repoService.gitClient.FileExists(syncCtx, repoPath, sync.ComposePath) {
+ if !s.repoService.gitClient.FileExists(ctx, repoPath, sync.ComposePath) {
errMsg := fmt.Sprintf("compose file not found: %s", sync.ComposePath)
- return result, s.failSync(syncCtx, id, result, sync, actor, fmt.Sprintf("Compose file not found at %s", sync.ComposePath), errMsg)
+ return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, fmt.Sprintf("Compose file not found at %s", sync.ComposePath), errMsg)
}
- // Read compose file content
- composeContent, err := s.repoService.gitClient.ReadFile(syncCtx, repoPath, sync.ComposePath)
+ composeContent, err := s.repoService.gitClient.ReadFile(ctx, repoPath, sync.ComposePath)
if err != nil {
- return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to read compose file", err.Error())
+ return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose file", err.Error())
+ }
+
+ source := &preparedSyncSource{
+ repoPath: repoPath,
+ commitHash: commitHash,
+ composeContent: composeContent,
}
- // Try to read .env file from the same directory as the compose file
- var envContent *string
envPath := filepath.Join(filepath.Dir(sync.ComposePath), ".env")
- if s.repoService.gitClient.FileExists(syncCtx, repoPath, envPath) {
- content, err := s.repoService.gitClient.ReadFile(syncCtx, repoPath, envPath)
+ if s.repoService.gitClient.FileExists(ctx, repoPath, envPath) {
+ content, err := s.repoService.gitClient.ReadFile(ctx, repoPath, envPath)
if err != nil {
- slog.WarnContext(syncCtx, "Failed to read .env file", "path", envPath, "error", err)
+ slog.WarnContext(ctx, "Failed to read .env file", "path", envPath, "error", err)
} else {
- envContent = &content
+ source.envContent = &content
}
}
- var project *models.Project
- var syncedFiles []string
+ return source, nil
+}
- if sync.SyncDirectory {
- // Directory sync mode - sync entire directory containing compose file
- slog.InfoContext(syncCtx, "Using directory sync mode", "syncId", id, "composePath", sync.ComposePath)
-
- // Walk directory once and get all files
- var dirComposeContent string
- var syncFiles []projects.SyncFile
- var err error
- dirComposeContent, syncFiles, err = s.walkAndParseSyncDirectory(syncCtx, sync, repoPath)
- if err != nil {
- return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to walk directory", err.Error())
- }
+// performDirectorySync runs the directory-sync path and only triggers a
+// redeploy when an already running project's synced contents changed.
+func (s *GitOpsSyncService) performDirectorySync(ctx context.Context, sync *models.GitOpsSync, id string, actor models.User, result *gitops.SyncResult, source *preparedSyncSource) (*gitops.SyncResult, error) {
+ slog.InfoContext(ctx, "Using directory sync mode", "syncId", id, "composePath", sync.ComposePath)
- // Get or create project with compose content
- project, err = s.getOrCreateProjectInternal(syncCtx, sync, id, dirComposeContent, nil, result, actor)
- if err != nil {
- return result, err
- }
+ _, syncFiles, err := s.walkAndParseSyncDirectory(ctx, sync, source.repoPath)
+ if err != nil {
+ return result, s.failSync(ctx, id, result, sync, actor, "Failed to walk directory", err.Error())
+ }
- // Write all directory files to the project
- oldSyncedFiles := parseSyncedFiles(sync.SyncedFiles)
- syncedFiles, err = s.writeSyncFilesToProject(syncCtx, sync, project, syncFiles, oldSyncedFiles)
- if err != nil {
- slog.ErrorContext(syncCtx, "Failed to write directory files to project", "error", err, "syncId", id)
- // Don't fail the sync - the project was created/updated with compose content
- // Fall back to just tracking the file paths
- syncedFiles = make([]string, len(syncFiles))
- for i, f := range syncFiles {
- syncedFiles[i] = f.RelativePath
- }
- }
+ project, syncedFiles, _, contentsChanged, err := s.syncProjectDirectoryInternal(ctx, sync, syncFiles, actor)
+ if err != nil {
+ return result, s.failSync(ctx, id, result, sync, actor, "Failed to sync project directory", err.Error())
+ }
- // Update sync status with synced files
- s.updateSyncStatusWithFiles(syncCtx, id, "success", "", commitHash, syncedFiles)
- } else {
- // Single file sync mode - existing behavior
- slog.InfoContext(syncCtx, "Using single file sync mode", "syncId", id, "composePath", sync.ComposePath)
+ if contentsChanged {
+ s.redeployIfRunningAfterSync(ctx, project, actor, "directory")
+ }
- // Get or create project (uses composeContent and envContent already read above)
- var err error
- project, err = s.getOrCreateProjectInternal(syncCtx, sync, id, composeContent, envContent, result, actor)
- if err != nil {
- return result, err
- }
+ s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)
+ result.Success = true
+ result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)
+ s.logSyncSuccess(ctx, sync, project, actor)
+ slog.InfoContext(ctx, "GitOps sync completed", "syncId", id, "project", project.Name)
- // Track single compose file as synced
- syncedFiles = []string{filepath.Base(sync.ComposePath)}
+ return result, nil
+}
- // Update sync status with synced files
- s.updateSyncStatusWithFiles(syncCtx, id, "success", "", commitHash, syncedFiles)
+// performSingleFileSync preserves the legacy compose-only Git sync behavior.
+func (s *GitOpsSyncService) performSingleFileSync(ctx context.Context, sync *models.GitOpsSync, id string, actor models.User, result *gitops.SyncResult, source *preparedSyncSource) (*gitops.SyncResult, error) {
+ slog.InfoContext(ctx, "Using single file sync mode", "syncId", id, "composePath", sync.ComposePath)
+
+ project, err := s.getOrCreateProjectInternal(ctx, sync, id, source.composeContent, source.envContent, result, actor)
+ if err != nil {
+ return result, err
}
+ syncedFiles := []string{filepath.Base(sync.ComposePath)}
+ s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)
result.Success = true
- if sync.SyncDirectory {
- result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)
- } else {
- result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)
+ result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)
+ s.logSyncSuccess(ctx, sync, project, actor)
+ slog.InfoContext(ctx, "GitOps sync completed", "syncId", id, "project", project.Name)
+
+ return result, nil
+}
+
+// redeployIfRunningAfterSync redeploys a project only when it is already
+// running and the latest sync actually changed managed content.
+func (s *GitOpsSyncService) redeployIfRunningAfterSync(ctx context.Context, project *models.Project, actor models.User, syncMode string) {
+ details, err := s.projectService.GetProjectDetails(ctx, project.ID)
+ if err != nil {
+ return
+ }
+ if details.Status != string(models.ProjectStatusRunning) && details.Status != string(models.ProjectStatusPartiallyRunning) {
+ return
}
- // Log success event
- _, _ = s.eventService.CreateEvent(syncCtx, CreateEventRequest{
+ slog.InfoContext(ctx, "Redeploying project due to content change from Git sync", "syncMode", syncMode, "projectName", project.Name, "projectId", project.ID)
+ if err := s.projectService.RedeployProject(ctx, project.ID, actor); err != nil {
+ slog.ErrorContext(ctx, "Failed to redeploy project after Git sync", "syncMode", syncMode, "error", err, "projectId", project.ID)
+ }
+}
+
+// logSyncSuccess records the Git sync completion event once the filesystem and
+// sync-status updates have already succeeded.
+func (s *GitOpsSyncService) logSyncSuccess(ctx context.Context, sync *models.GitOpsSync, project *models.Project, actor models.User) {
+ _, _ = s.eventService.CreateEvent(ctx, CreateEventRequest{
Type: models.EventTypeGitSyncRun,
Severity: models.EventSeveritySuccess,
Title: "Git sync completed",
@@ -586,10 +631,6 @@ func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id s
Username: new(actor.Username),
EnvironmentID: new(sync.EnvironmentID),
})
-
- slog.InfoContext(syncCtx, "GitOps sync completed", "syncId", id, "project", project.Name)
-
- return result, nil
}
func (s *GitOpsSyncService) updateSyncStatus(ctx context.Context, id, status, errorMsg, commitHash string) {
@@ -803,12 +844,6 @@ func (s *GitOpsSyncService) createProjectForSyncInternal(ctx context.Context, sy
slog.InfoContext(ctx, "Created project for GitOps sync", "projectName", sync.ProjectName, "projectId", project.ID)
- // Deploy the project immediately after creation
- slog.InfoContext(ctx, "Deploying project after initial Git sync", "projectName", project.Name, "projectId", project.ID)
- if err := s.projectService.DeployProject(ctx, project.ID, actor, nil); err != nil {
- slog.ErrorContext(ctx, "Failed to deploy project after initial Git sync", "error", err, "projectId", project.ID)
- }
-
return project, nil
}
@@ -872,12 +907,6 @@ func envContentChangedInternal(oldEnv, newEnv string) bool {
return !maps.Equal(oldEnvMap, newEnvMap)
}
-// getProjectsDirectory returns the configured projects directory path
-func (s *GitOpsSyncService) getProjectsDirectory(ctx context.Context) (string, error) {
- projectsDirSetting := s.settingsService.GetStringSetting(ctx, "projectsDirectory", "/app/data/projects")
- return projects.GetProjectsDirectory(ctx, strings.TrimSpace(projectsDirSetting))
-}
-
// parseSyncedFiles parses the JSON array of synced file paths from the database
func parseSyncedFiles(syncedFilesJSON *string) []string {
if syncedFilesJSON == nil || *syncedFilesJSON == "" {
@@ -946,36 +975,401 @@ func (s *GitOpsSyncService) walkAndParseSyncDirectory(ctx context.Context, sync
return composeContent, syncFiles, nil
}
-// writeSyncFilesToProject writes the given sync files to the project directory.
-// If oldSyncedFiles is provided, removed files will be cleaned up first.
-func (s *GitOpsSyncService) writeSyncFilesToProject(ctx context.Context, sync *models.GitOpsSync, project *models.Project, syncFiles []projects.SyncFile, oldSyncedFiles []string) ([]string, error) {
- projectsDir, err := s.getProjectsDirectory(ctx)
+// syncProjectDirectoryInternal runs the new directory-sync path end to end:
+// stage files, validate the staged tree, then create or update the project.
+func (s *GitOpsSyncService) syncProjectDirectoryInternal(ctx context.Context, sync *models.GitOpsSync, syncFiles []projects.SyncFile, actor models.User) (*models.Project, []string, bool, bool, error) {
+ stage, err := s.stageDirectorySyncInternal(ctx, sync, syncFiles)
+ if err != nil {
+ return nil, nil, false, false, err
+ }
+ defer func() {
+ if stage != nil && stage.stagePath != "" {
+ _ = os.RemoveAll(stage.stagePath)
+ }
+ }()
+
+ if stage.project == nil {
+ project, err := s.createDirectorySyncProjectInternal(ctx, sync, stage, actor)
+ if err != nil {
+ return nil, nil, false, false, err
+ }
+ return project, stage.syncedFiles, true, true, nil
+ }
+
+ project, err := s.updateDirectorySyncProjectInternal(ctx, sync, stage)
+ if err != nil {
+ return nil, nil, false, false, err
+ }
+ return project, stage.syncedFiles, false, stage.contentsChanged, nil
+}
+
+// stageDirectorySyncInternal builds a temporary project tree that reflects the exact
+// repo layout after sync, including cleanup of files removed from the repo.
+func (s *GitOpsSyncService) stageDirectorySyncInternal(ctx context.Context, sync *models.GitOpsSync, syncFiles []projects.SyncFile) (*stagedDirectorySync, error) {
+ projectsDir, err := s.projectService.getProjectsDirectoryInternal(ctx)
if err != nil {
return nil, fmt.Errorf("failed to get projects directory: %w", err)
}
- // Build list of new file paths
- newFiles := make([]string, len(syncFiles))
- for i, f := range syncFiles {
- newFiles[i] = f.RelativePath
+ stagePath, err := os.MkdirTemp(projectsDir, ".gitops-sync-stage-*")
+ if err != nil {
+ return nil, fmt.Errorf("failed to create staging directory: %w", err)
+ }
+
+ project, err := s.getDirectorySyncProjectInternal(ctx, sync)
+ if err != nil {
+ _ = os.RemoveAll(stagePath)
+ return nil, err
+ }
+
+ if project != nil {
+ if err := copyDirectoryContentsInternal(project.Path, stagePath); err != nil {
+ _ = os.RemoveAll(stagePath)
+ return nil, fmt.Errorf("failed to stage current project files: %w", err)
+ }
}
- // Clean up removed files if we have old sync data
+ syncedFiles := make([]string, len(syncFiles))
+ for i, file := range syncFiles {
+ syncedFiles[i] = file.RelativePath
+ }
+
+ oldSyncedFiles := parseSyncedFiles(sync.SyncedFiles)
if len(oldSyncedFiles) > 0 {
- if err := projects.CleanupRemovedFiles(projectsDir, project.Path, oldSyncedFiles, newFiles); err != nil {
- slog.WarnContext(ctx, "Failed to cleanup removed files", "error", err, "syncId", sync.ID)
- // Continue despite cleanup error - it's best effort
+ if err := projects.CleanupRemovedFiles(projectsDir, stagePath, oldSyncedFiles, syncedFiles); err != nil {
+ _ = os.RemoveAll(stagePath)
+ return nil, fmt.Errorf("failed to clean removed synced files: %w", err)
}
}
- // Write all files to project directory
- writtenPaths, err := projects.WriteSyncedDirectory(projectsDir, project.Path, syncFiles)
+ composeFileName := filepath.Base(sync.ComposePath)
+ if err := removeStaleComposeFilesInternal(stagePath, composeFileName, syncedFiles); err != nil {
+ _ = os.RemoveAll(stagePath)
+ return nil, fmt.Errorf("failed to remove stale compose files: %w", err)
+ }
+
+ contentsChanged, err := directorySyncContentsChangedInternal(project, syncFiles, oldSyncedFiles, composeFileName)
+ if err != nil {
+ _ = os.RemoveAll(stagePath)
+ return nil, fmt.Errorf("failed to compare staged directory changes: %w", err)
+ }
+
+ // Write the repo files after cleanup so validation sees the final on-disk
+ // tree exactly as it will exist in the managed project.
+ if _, err := projects.WriteSyncedDirectory(projectsDir, stagePath, syncFiles); err != nil {
+ _ = os.RemoveAll(stagePath)
+ return nil, fmt.Errorf("failed to write staged sync files: %w", err)
+ }
+
+ serviceCount, err := s.validateDirectorySyncStageInternal(ctx, sync.ProjectName, stagePath, composeFileName)
+ if err != nil {
+ _ = os.RemoveAll(stagePath)
+ return nil, fmt.Errorf("invalid compose file: %w", err)
+ }
+
+ return &stagedDirectorySync{
+ stagePath: stagePath,
+ composeFileName: composeFileName,
+ project: project,
+ syncedFiles: syncedFiles,
+ serviceCount: serviceCount,
+ contentsChanged: contentsChanged,
+ }, nil
+}
+
+// getDirectorySyncProjectInternal resolves the linked project for a sync when one
+// exists, while tolerating deleted/stale project references.
+func (s *GitOpsSyncService) getDirectorySyncProjectInternal(ctx context.Context, sync *models.GitOpsSync) (*models.Project, error) {
+ if sync.ProjectID == nil || *sync.ProjectID == "" {
+ return nil, nil
+ }
+
+ project, err := s.projectService.GetProjectFromDatabaseByID(ctx, *sync.ProjectID)
if err != nil {
- return nil, fmt.Errorf("failed to write synced directory: %w", err)
+ slog.WarnContext(ctx, "Existing project not found, will create new one", "projectId", *sync.ProjectID, "error", err)
+ return nil, nil
}
- slog.InfoContext(ctx, "Directory sync written", "syncId", sync.ID, "projectId", project.ID, "filesWritten", len(writtenPaths))
- return writtenPaths, nil
+ if err := s.projectService.ensureProjectPathUnderRoot(ctx, project, false); err != nil {
+ return nil, err
+ }
+
+ return project, nil
+}
+
+// validateDirectorySyncStageInternal loads the staged compose project using the real
+// synced compose filename so include/env_file resolution happens against the
+// fully copied directory contents.
+func (s *GitOpsSyncService) validateDirectorySyncStageInternal(ctx context.Context, projectName, stagePath, composeFileName string) (int, error) {
+ projectsDir, err := s.projectService.getProjectsDirectoryInternal(ctx)
+ if err != nil {
+ return 0, err
+ }
+
+ pathMapper, pmErr := s.projectService.getPathMapper(ctx)
+ if pmErr != nil {
+ slog.WarnContext(ctx, "failed to create path mapper for directory sync validation, continuing without translation", "error", pmErr)
+ }
+
+ autoInjectEnv := s.settingsService.GetBoolSetting(ctx, "autoInjectEnv", false)
+ project, err := projects.LoadComposeProject(
+ ctx,
+ filepath.Join(stagePath, composeFileName),
+ normalizeComposeProjectName(projectName),
+ projectsDir,
+ autoInjectEnv,
+ pathMapper,
+ )
+ if err != nil {
+ return 0, err
+ }
+
+ return len(project.Services), nil
+}
+
+// createDirectorySyncProjectInternal promotes a validated staged tree into a new
+// managed project directory and links it back to the Git sync record.
+func (s *GitOpsSyncService) createDirectorySyncProjectInternal(ctx context.Context, sync *models.GitOpsSync, stage *stagedDirectorySync, actor models.User) (*models.Project, error) {
+ projectsDir, err := s.projectService.getProjectsDirectoryInternal(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get projects directory: %w", err)
+ }
+
+ basePath := filepath.Join(projectsDir, projects.SanitizeProjectName(sync.ProjectName))
+ projectPath, folderName, err := projects.CreateUniqueDir(projectsDir, basePath, sync.ProjectName, 0o755)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create project directory: %w", err)
+ }
+
+ if err := os.Remove(projectPath); err != nil {
+ return nil, fmt.Errorf("failed to prepare project directory: %w", err)
+ }
+
+ if err := os.Rename(stage.stagePath, projectPath); err != nil {
+ return nil, fmt.Errorf("failed to promote staged project directory: %w", err)
+ }
+ stage.stagePath = ""
+
+ project := &models.Project{
+ Name: sync.ProjectName,
+ DirName: new(folderName),
+ Path: projectPath,
+ Status: models.ProjectStatusStopped,
+ ServiceCount: stage.serviceCount,
+ RunningCount: 0,
+ }
+
+ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.Create(project).Error; err != nil {
+ return fmt.Errorf("failed to create project: %w", err)
+ }
+
+ if err := tx.Model(&models.GitOpsSync{}).Where("id = ?", sync.ID).Update("project_id", project.ID).Error; err != nil {
+ return fmt.Errorf("failed to update sync with project ID: %w", err)
+ }
+
+ if err := tx.Model(&models.Project{}).Where("id = ?", project.ID).Update("gitops_managed_by", sync.ID).Error; err != nil {
+ return fmt.Errorf("failed to mark project as GitOps-managed: %w", err)
+ }
+
+ return nil
+ }); err != nil {
+ _ = os.RemoveAll(projectPath)
+ return nil, err
+ }
+
+ sync.ProjectID = &project.ID
+ s.projectService.cacheComposeProjectIDInternal(normalizeComposeProjectName(project.Name), project.ID)
+
+ if s.projectService.eventService != nil {
+ metadata := models.JSON{"action": "create", "projectID": project.ID, "projectName": project.Name, "path": projectPath}
+ if logErr := s.projectService.eventService.LogProjectEvent(ctx, models.EventTypeProjectCreate, project.ID, project.Name, actor.ID, actor.Username, "0", metadata); logErr != nil {
+ slog.ErrorContext(ctx, "could not log project creation", "error", logErr)
+ }
+ }
+
+ return project, nil
+}
+
+// updateDirectorySyncProjectInternal swaps a validated staged tree into the existing
+// project path with a temporary backup so failed promotion can roll back.
+func (s *GitOpsSyncService) updateDirectorySyncProjectInternal(ctx context.Context, sync *models.GitOpsSync, stage *stagedDirectorySync) (*models.Project, error) {
+ project := stage.project
+ projectPath := filepath.Clean(project.Path)
+ backupPath := ""
+
+ if info, err := os.Stat(projectPath); err == nil {
+ if !info.IsDir() {
+ return nil, fmt.Errorf("project path is not a directory: %s", projectPath)
+ }
+ backupPath = fmt.Sprintf("%s.gitops-backup-%d", projectPath, time.Now().UnixNano())
+ if err := os.Rename(projectPath, backupPath); err != nil {
+ return nil, fmt.Errorf("failed to move current project directory out of the way: %w", err)
+ }
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return nil, fmt.Errorf("failed to inspect current project directory: %w", err)
+ }
+
+ if err := os.Rename(stage.stagePath, projectPath); err != nil {
+ if backupPath != "" {
+ _ = os.Rename(backupPath, projectPath)
+ }
+ return nil, fmt.Errorf("failed to promote staged project directory: %w", err)
+ }
+ stage.stagePath = ""
+
+ if err := s.db.WithContext(ctx).Model(&models.Project{}).Where("id = ?", project.ID).Updates(map[string]any{
+ "service_count": stage.serviceCount,
+ "gitops_managed_by": sync.ID,
+ "updated_at": time.Now(),
+ }).Error; err != nil {
+ if backupPath != "" {
+ _ = os.RemoveAll(projectPath)
+ _ = os.Rename(backupPath, projectPath)
+ }
+ return nil, fmt.Errorf("failed to update project metadata after directory sync: %w", err)
+ }
+
+ if backupPath != "" {
+ _ = os.RemoveAll(backupPath)
+ }
+
+ return project, nil
+}
+
+// directorySyncContentsChangedInternal compares the currently managed synced files with
+// the newly fetched repo files to decide whether a redeploy is needed.
+func directorySyncContentsChangedInternal(project *models.Project, syncFiles []projects.SyncFile, oldSyncedFiles []string, composeFileName string) (bool, error) {
+ if project == nil {
+ return true, nil
+ }
+
+ if info, err := os.Stat(project.Path); err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return true, nil
+ }
+ return false, err
+ } else if !info.IsDir() {
+ return false, fmt.Errorf("project path is not a directory: %s", project.Path)
+ }
+
+ newFileSet := make(map[string]struct{}, len(syncFiles))
+ for _, file := range syncFiles {
+ newFileSet[file.RelativePath] = struct{}{}
+ existingContent, err := os.ReadFile(filepath.Join(project.Path, file.RelativePath))
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return true, nil
+ }
+ return false, err
+ }
+ if !bytes.Equal(existingContent, file.Content) {
+ return true, nil
+ }
+ }
+
+ for _, oldFile := range oldSyncedFiles {
+ if _, exists := newFileSet[oldFile]; exists {
+ continue
+ }
+ if _, err := os.Stat(filepath.Join(project.Path, oldFile)); err == nil {
+ return true, nil
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return false, err
+ }
+ }
+
+ for _, candidate := range projects.ComposeFileCandidates() {
+ if candidate == composeFileName {
+ continue
+ }
+ if _, exists := newFileSet[candidate]; exists {
+ continue
+ }
+ if _, err := os.Stat(filepath.Join(project.Path, candidate)); err == nil {
+ return true, nil
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return false, err
+ }
+ }
+
+ return false, nil
+}
+
+// removeStaleComposeFilesInternal drops old canonical compose filenames from the staged
+// project when the repo now uses a different compose filename.
+func removeStaleComposeFilesInternal(projectPath, composeFileName string, syncedFiles []string) error {
+ syncedFileSet := make(map[string]struct{}, len(syncedFiles))
+ for _, file := range syncedFiles {
+ syncedFileSet[file] = struct{}{}
+ }
+
+ for _, candidate := range projects.ComposeFileCandidates() {
+ if candidate == composeFileName {
+ continue
+ }
+ if _, exists := syncedFileSet[candidate]; exists {
+ continue
+ }
+ if err := os.Remove(filepath.Join(projectPath, candidate)); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// copyDirectoryContentsInternal clones the current project tree into the staging
+// directory so directory sync updates can preserve non-synced local files.
+func copyDirectoryContentsInternal(srcDir, destDir string) error {
+ srcRoot, err := os.OpenRoot(srcDir)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = srcRoot.Close() }()
+
+ destRoot, err := os.OpenRoot(destDir)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = destRoot.Close() }()
+
+ return filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error {
+ if walkErr != nil {
+ return walkErr
+ }
+ if path == srcDir {
+ return nil
+ }
+
+ relPath, err := filepath.Rel(srcDir, path)
+ if err != nil {
+ return err
+ }
+
+ if d.IsDir() {
+ return destRoot.MkdirAll(relPath, 0o755)
+ }
+ if d.Type()&os.ModeSymlink != 0 {
+ return nil
+ }
+
+ info, err := d.Info()
+ if err != nil {
+ return err
+ }
+
+ content, err := srcRoot.ReadFile(relPath)
+ if err != nil {
+ return err
+ }
+
+ if err := destRoot.MkdirAll(filepath.Dir(relPath), 0o755); err != nil {
+ return err
+ }
+
+ return destRoot.WriteFile(relPath, content, info.Mode())
+ })
}
// updateSyncStatusWithFiles updates sync status including the list of synced files
diff --git a/backend/internal/services/gitops_sync_service_test.go b/backend/internal/services/gitops_sync_service_test.go
index a1939e03a..8f9b15151 100644
--- a/backend/internal/services/gitops_sync_service_test.go
+++ b/backend/internal/services/gitops_sync_service_test.go
@@ -2,13 +2,246 @@ package services
import (
"context"
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
"testing"
+ "github.com/getarcaneapp/arcane/backend/internal/database"
"github.com/getarcaneapp/arcane/backend/internal/models"
+ "github.com/getarcaneapp/arcane/backend/pkg/projects"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "gorm.io/gorm"
)
+func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *database.DB, string) {
+ t.Helper()
+
+ ctx := context.Background()
+ db := setupProjectTestDB(t)
+ require.NoError(t, db.AutoMigrate(&models.GitOpsSync{}))
+
+ settingsService, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ projectsDir := t.TempDir()
+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))
+
+ projectService := NewProjectService(db, settingsService, nil, nil, nil, nil)
+
+ return NewGitOpsSyncService(db, nil, projectService, nil, settingsService), db, projectsDir
+}
+
+func TestGitOpsSyncService_SyncProjectDirectory_CreatesProjectPreservingRepoLayout(t *testing.T) {
+ ctx := context.Background()
+ svc, db, _ := setupGitOpsSyncDirectoryTestService(t)
+
+ sync := &models.GitOpsSync{
+ BaseModel: models.BaseModel{ID: "sync-directory-create"},
+ Name: "demo-sync",
+ EnvironmentID: "0",
+ RepositoryID: "repo-1",
+ ComposePath: "apps/demo/docker-compose.yaml",
+ ProjectName: "demo-project",
+ SyncDirectory: true,
+ }
+ require.NoError(t, db.Create(sync).Error)
+
+ syncFiles := []projects.SyncFile{
+ {
+ RelativePath: "docker-compose.yaml",
+ Content: []byte(`include:
+ - meta.yaml
+services:
+ app:
+ image: nginx:alpine
+ env_file:
+ - .env
+`),
+ },
+ {
+ RelativePath: "meta.yaml",
+ Content: []byte(`services:
+ helper:
+ image: busybox:latest
+`),
+ },
+ {
+ RelativePath: ".env",
+ Content: []byte("APP_MODE=production\n"),
+ },
+ }
+
+ project, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, models.User{})
+ require.NoError(t, err)
+ require.NotNil(t, project)
+ require.True(t, created)
+ require.True(t, changed)
+ require.ElementsMatch(t, []string{"docker-compose.yaml", "meta.yaml", ".env"}, syncedFiles)
+
+ composePath, detectErr := projects.DetectComposeFile(project.Path)
+ require.NoError(t, detectErr)
+ assert.Equal(t, filepath.Join(project.Path, "docker-compose.yaml"), composePath)
+
+ composeBytes, err := os.ReadFile(filepath.Join(project.Path, "docker-compose.yaml"))
+ require.NoError(t, err)
+ assert.Contains(t, string(composeBytes), "include:")
+
+ metaBytes, err := os.ReadFile(filepath.Join(project.Path, "meta.yaml"))
+ require.NoError(t, err)
+ assert.Contains(t, string(metaBytes), "helper:")
+
+ envBytes, err := os.ReadFile(filepath.Join(project.Path, ".env"))
+ require.NoError(t, err)
+ assert.Equal(t, "APP_MODE=production\n", string(envBytes))
+
+ _, statErr := os.Stat(filepath.Join(project.Path, "compose.yaml"))
+ assert.ErrorIs(t, statErr, os.ErrNotExist)
+}
+
+func TestGitOpsSyncService_SyncProjectDirectory_UpdatesProjectAndCleansOldSyncedFiles(t *testing.T) {
+ ctx := context.Background()
+ svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)
+
+ projectPath := filepath.Join(projectsDir, "demo-project")
+ require.NoError(t, os.MkdirAll(filepath.Join(projectPath, "nested"), 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "docker-compose.yaml"), []byte(`include:
+ - meta.yaml
+services:
+ app:
+ image: nginx:1.26-alpine
+`), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "meta.yaml"), []byte(`services:
+ helper:
+ image: busybox:1.36
+`), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "old.txt"), []byte("remove me\n"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "keep.txt"), []byte("keep me\n"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services: {}\n"), 0o644))
+
+ project := &models.Project{
+ BaseModel: models.BaseModel{ID: "proj-directory-update"},
+ Name: "demo-project",
+ DirName: new("demo-project"),
+ Path: projectPath,
+ Status: models.ProjectStatusStopped,
+ }
+ require.NoError(t, db.Create(project).Error)
+
+ oldSyncedFilesJSON, err := json.Marshal([]string{"docker-compose.yaml", "meta.yaml", "old.txt"})
+ require.NoError(t, err)
+
+ sync := &models.GitOpsSync{
+ BaseModel: models.BaseModel{ID: "sync-directory-update"},
+ Name: "demo-sync",
+ EnvironmentID: "0",
+ RepositoryID: "repo-1",
+ ComposePath: "apps/demo/docker-compose.yaml",
+ ProjectName: "demo-project",
+ ProjectID: &project.ID,
+ SyncDirectory: true,
+ SyncedFiles: new(string(oldSyncedFilesJSON)),
+ }
+ require.NoError(t, db.Create(sync).Error)
+
+ syncFiles := []projects.SyncFile{
+ {
+ RelativePath: "docker-compose.yaml",
+ Content: []byte(`include:
+ - nested/feature.yaml
+services:
+ app:
+ image: nginx:1.27-alpine
+`),
+ },
+ {
+ RelativePath: "nested/feature.yaml",
+ Content: []byte(`services:
+ worker:
+ image: busybox:latest
+`),
+ },
+ }
+
+ updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, models.User{})
+ require.NoError(t, err)
+ require.NotNil(t, updatedProject)
+ require.False(t, created)
+ require.True(t, changed)
+ require.ElementsMatch(t, []string{"docker-compose.yaml", "nested/feature.yaml"}, syncedFiles)
+
+ composePath, detectErr := projects.DetectComposeFile(updatedProject.Path)
+ require.NoError(t, detectErr)
+ assert.Equal(t, filepath.Join(updatedProject.Path, "docker-compose.yaml"), composePath)
+
+ _, statErr := os.Stat(filepath.Join(updatedProject.Path, "old.txt"))
+ assert.ErrorIs(t, statErr, os.ErrNotExist)
+
+ _, statErr = os.Stat(filepath.Join(updatedProject.Path, "compose.yaml"))
+ assert.ErrorIs(t, statErr, os.ErrNotExist)
+
+ keepBytes, err := os.ReadFile(filepath.Join(updatedProject.Path, "keep.txt"))
+ require.NoError(t, err)
+ assert.Equal(t, "keep me\n", string(keepBytes))
+
+ featureBytes, err := os.ReadFile(filepath.Join(updatedProject.Path, "nested", "feature.yaml"))
+ require.NoError(t, err)
+ assert.Contains(t, string(featureBytes), "worker:")
+}
+
+func TestGitOpsSyncService_CreateDirectorySyncProjectInternal_RollsBackProjectOnUpdateFailure(t *testing.T) {
+ ctx := context.Background()
+ svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)
+
+ sync := &models.GitOpsSync{
+ BaseModel: models.BaseModel{ID: "sync-directory-tx-rollback"},
+ Name: "demo-sync",
+ EnvironmentID: "0",
+ RepositoryID: "repo-1",
+ ComposePath: "apps/demo/docker-compose.yaml",
+ ProjectName: "demo-project",
+ SyncDirectory: true,
+ }
+ require.NoError(t, db.Create(sync).Error)
+
+ stagePath := t.TempDir()
+ require.NoError(t, os.WriteFile(filepath.Join(stagePath, "docker-compose.yaml"), []byte("services: {}\n"), 0o644))
+
+ stage := &stagedDirectorySync{
+ stagePath: stagePath,
+ composeFileName: "docker-compose.yaml",
+ serviceCount: 1,
+ }
+
+ callbackName := "test:fail_project_gitops_update"
+ require.NoError(t, db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {
+ if tx.Statement != nil && tx.Statement.Table == "projects" {
+ tx.AddError(errors.New("forced project update failure"))
+ }
+ }))
+ defer func() {
+ db.Callback().Update().Remove(callbackName)
+ }()
+
+ project, err := svc.createDirectorySyncProjectInternal(ctx, sync, stage, models.User{})
+ require.Error(t, err)
+ require.Nil(t, project)
+ assert.Contains(t, err.Error(), "failed to mark project as GitOps-managed")
+
+ var projectCount int64
+ require.NoError(t, db.Model(&models.Project{}).Count(&projectCount).Error)
+ assert.Zero(t, projectCount)
+
+ var storedSync models.GitOpsSync
+ require.NoError(t, db.First(&storedSync, "id = ?", sync.ID).Error)
+ assert.Nil(t, storedSync.ProjectID)
+
+ _, statErr := os.Stat(filepath.Join(projectsDir, "demo-project"))
+ assert.ErrorIs(t, statErr, os.ErrNotExist)
+}
+
func TestEnvContentChangedInternal(t *testing.T) {
t.Run("ignores formatting-only changes", func(t *testing.T) {
oldEnv := "B=2\nA=1\n# comment\n"
diff --git a/backend/pkg/projects/fs_writer.go b/backend/pkg/projects/fs_writer.go
index 2dc724df8..c8e66aa43 100644
--- a/backend/pkg/projects/fs_writer.go
+++ b/backend/pkg/projects/fs_writer.go
@@ -20,6 +20,12 @@ var composeFileCandidates = []string{
"podman-compose.yml",
}
+// ComposeFileCandidates returns the supported compose filenames in Arcane's
+// detection order. A copy is returned so callers can't mutate package state.
+func ComposeFileCandidates() []string {
+ return append([]string(nil), composeFileCandidates...)
+}
+
// detectExistingComposeFile finds an existing compose file in the directory
func detectExistingComposeFile(dir string) string {
for _, filename := range composeFileCandidates {