Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: git sync is not pulling full folders, no longer deploy on creation of sync (#2239)
Kyle Mendell6ceefc826747
diff --git a/backend/internal/services/gitops_sync_service.go b/backend/internal/services/gitops_sync_service.goindex 2f3f03fa4..ab3043bd3 100644--- a/backend/internal/services/gitops_sync_service.go+++ b/backend/internal/services/gitops_sync_service.go@@ -1,12 +1,14 @@package servicesimport (+ "bytes""context""encoding/json""errors""fmt""log/slog""maps"+ "os""path/filepath""strings""time"@@ -46,6 +48,26 @@ type scheduledGitOpsSync struct {LastSyncAt *time.Time}+// preparedSyncSource captures the repository data needed by the sync execution+// paths after the source repository has been cloned and validated.+type preparedSyncSource struct {+ repoPath string+ commitHash string+ composeContent string+ envContent *string+}++// stagedDirectorySync holds the fully prepared directory-sync result before it+// is promoted into the live project path.+type stagedDirectorySync struct {+ stagePath string+ composeFileName string+ project *models.Project+ syncedFiles []string+ serviceCount int+ contentsChanged bool+}+func validateSyncLimits(maxFiles *int, maxTotalSize, maxBinarySize *int64) error {if maxFiles != nil && *maxFiles < 0 {return fmt.Errorf("maxSyncFiles must be non-negative")@@ -458,123 +480,146 @@ func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id sSyncedAt: time.Now(),}- // Get repository and auth config+ source, err := s.prepareSyncSource(syncCtx, sync, result, actor)+ if source != nil && source.repoPath != "" {+ defer func() {+ if cleanupErr := s.repoService.gitClient.Cleanup(source.repoPath); cleanupErr != nil {+ slog.WarnContext(syncCtx, "Failed to cleanup repository", "path", source.repoPath, "error", cleanupErr)+ }+ }()+ }+ if err != nil {+ return result, err+ }++ if sync.SyncDirectory {+ return s.performDirectorySync(syncCtx, sync, id, actor, result, source)+ }++ return s.performSingleFileSync(syncCtx, sync, id, actor, result, source)+}++// prepareSyncSource clones the source repository, validates that the configured+// compose file exists, and reads the compose/env inputs for the sync flow.+func (s *GitOpsSyncService) prepareSyncSource(ctx context.Context, sync *models.GitOpsSync, result *gitops.SyncResult, actor models.User) (*preparedSyncSource, error) {repository := sync.Repositoryif repository == nil {- return result, s.failSync(syncCtx, id, result, sync, actor, "Repository not found", "repository not found")+ return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Repository not found", "repository not found")}- authConfig, err := s.repoService.GetAuthConfig(syncCtx, repository)+ authConfig, err := s.repoService.GetAuthConfig(ctx, repository)if err != nil {- return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to get authentication config", err.Error())+ return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to get authentication config", err.Error())}- // Clone the repository- repoPath, err := s.repoService.gitClient.Clone(syncCtx, repository.URL, sync.Branch, authConfig)+ repoPath, err := s.repoService.gitClient.Clone(ctx, repository.URL, sync.Branch, authConfig)if err != nil {- return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to clone repository", err.Error())+ return nil, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to clone repository", err.Error())}- defer func() {- if cleanupErr := s.repoService.gitClient.Cleanup(repoPath); cleanupErr != nil {- slog.WarnContext(syncCtx, "Failed to cleanup repository", "path", repoPath, "error", cleanupErr)- }- }()- // Get the current commit hash- commitHash, err := s.repoService.gitClient.GetCurrentCommit(syncCtx, repoPath)+ commitHash, err := s.repoService.gitClient.GetCurrentCommit(ctx, repoPath)if err != nil {- slog.WarnContext(syncCtx, "Failed to get commit hash", "error", err)+ slog.WarnContext(ctx, "Failed to get commit hash", "error", err)commitHash = ""}- // Check if compose file exists- if !s.repoService.gitClient.FileExists(syncCtx, repoPath, sync.ComposePath) {+ if !s.repoService.gitClient.FileExists(ctx, repoPath, sync.ComposePath) {errMsg := fmt.Sprintf("compose file not found: %s", sync.ComposePath)- return result, s.failSync(syncCtx, id, result, sync, actor, fmt.Sprintf("Compose file not found at %s", sync.ComposePath), errMsg)+ return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, fmt.Sprintf("Compose file not found at %s", sync.ComposePath), errMsg)}- // Read compose file content- composeContent, err := s.repoService.gitClient.ReadFile(syncCtx, repoPath, sync.ComposePath)+ composeContent, err := s.repoService.gitClient.ReadFile(ctx, repoPath, sync.ComposePath)if err != nil {- return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to read compose file", err.Error())+ return &preparedSyncSource{repoPath: repoPath, commitHash: commitHash}, s.failSync(ctx, sync.ID, result, sync, actor, "Failed to read compose file", err.Error())+ }++ source := &preparedSyncSource{+ repoPath: repoPath,+ commitHash: commitHash,+ composeContent: composeContent,}- // Try to read .env file from the same directory as the compose file- var envContent *stringenvPath := filepath.Join(filepath.Dir(sync.ComposePath), ".env")- if s.repoService.gitClient.FileExists(syncCtx, repoPath, envPath) {- content, err := s.repoService.gitClient.ReadFile(syncCtx, repoPath, envPath)+ if s.repoService.gitClient.FileExists(ctx, repoPath, envPath) {+ content, err := s.repoService.gitClient.ReadFile(ctx, repoPath, envPath)if err != nil {- slog.WarnContext(syncCtx, "Failed to read .env file", "path", envPath, "error", err)+ slog.WarnContext(ctx, "Failed to read .env file", "path", envPath, "error", err)} else {- envContent = &content+ source.envContent = &content}}- var project *models.Project- var syncedFiles []string+ return source, nil+}- if sync.SyncDirectory {- // Directory sync mode - sync entire directory containing compose file- slog.InfoContext(syncCtx, "Using directory sync mode", "syncId", id, "composePath", sync.ComposePath)-- // Walk directory once and get all files- var dirComposeContent string- var syncFiles []projects.SyncFile- var err error- dirComposeContent, syncFiles, err = s.walkAndParseSyncDirectory(syncCtx, sync, repoPath)- if err != nil {- return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to walk directory", err.Error())- }+// performDirectorySync runs the directory-sync path and only triggers a+// redeploy when an already running project's synced contents changed.+func (s *GitOpsSyncService) performDirectorySync(ctx context.Context, sync *models.GitOpsSync, id string, actor models.User, result *gitops.SyncResult, source *preparedSyncSource) (*gitops.SyncResult, error) {+ slog.InfoContext(ctx, "Using directory sync mode", "syncId", id, "composePath", sync.ComposePath)- // Get or create project with compose content- project, err = s.getOrCreateProjectInternal(syncCtx, sync, id, dirComposeContent, nil, result, actor)- if err != nil {- return result, err- }+ _, syncFiles, err := s.walkAndParseSyncDirectory(ctx, sync, source.repoPath)+ if err != nil {+ return result, s.failSync(ctx, id, result, sync, actor, "Failed to walk directory", err.Error())+ }- // Write all directory files to the project- oldSyncedFiles := parseSyncedFiles(sync.SyncedFiles)- syncedFiles, err = s.writeSyncFilesToProject(syncCtx, sync, project, syncFiles, oldSyncedFiles)- if err != nil {- slog.ErrorContext(syncCtx, "Failed to write directory files to project", "error", err, "syncId", id)- // Don't fail the sync - the project was created/updated with compose content- // Fall back to just tracking the file paths- syncedFiles = make([]string, len(syncFiles))- for i, f := range syncFiles {- syncedFiles[i] = f.RelativePath- }- }+ project, syncedFiles, _, contentsChanged, err := s.syncProjectDirectoryInternal(ctx, sync, syncFiles, actor)+ if err != nil {+ return result, s.failSync(ctx, id, result, sync, actor, "Failed to sync project directory", err.Error())+ }- // Update sync status with synced files- s.updateSyncStatusWithFiles(syncCtx, id, "success", "", commitHash, syncedFiles)- } else {- // Single file sync mode - existing behavior- slog.InfoContext(syncCtx, "Using single file sync mode", "syncId", id, "composePath", sync.ComposePath)+ if contentsChanged {+ s.redeployIfRunningAfterSync(ctx, project, actor, "directory")+ }- // Get or create project (uses composeContent and envContent already read above)- var err error- project, err = s.getOrCreateProjectInternal(syncCtx, sync, id, composeContent, envContent, result, actor)- if err != nil {- return result, err- }+ s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)+ result.Success = true+ result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)+ s.logSyncSuccess(ctx, sync, project, actor)+ slog.InfoContext(ctx, "GitOps sync completed", "syncId", id, "project", project.Name)- // Track single compose file as synced- syncedFiles = []string{filepath.Base(sync.ComposePath)}+ return result, nil+}- // Update sync status with synced files- s.updateSyncStatusWithFiles(syncCtx, id, "success", "", commitHash, syncedFiles)+// performSingleFileSync preserves the legacy compose-only Git sync behavior.+func (s *GitOpsSyncService) performSingleFileSync(ctx context.Context, sync *models.GitOpsSync, id string, actor models.User, result *gitops.SyncResult, source *preparedSyncSource) (*gitops.SyncResult, error) {+ slog.InfoContext(ctx, "Using single file sync mode", "syncId", id, "composePath", sync.ComposePath)++ project, err := s.getOrCreateProjectInternal(ctx, sync, id, source.composeContent, source.envContent, result, actor)+ if err != nil {+ return result, err}+ syncedFiles := []string{filepath.Base(sync.ComposePath)}+ s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)result.Success = true- if sync.SyncDirectory {- result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)- } else {- result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)+ result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)+ s.logSyncSuccess(ctx, sync, project, actor)+ slog.InfoContext(ctx, "GitOps sync completed", "syncId", id, "project", project.Name)++ return result, nil+}++// redeployIfRunningAfterSync redeploys a project only when it is already+// running and the latest sync actually changed managed content.+func (s *GitOpsSyncService) redeployIfRunningAfterSync(ctx context.Context, project *models.Project, actor models.User, syncMode string) {+ details, err := s.projectService.GetProjectDetails(ctx, project.ID)+ if err != nil {+ return+ }+ if details.Status != string(models.ProjectStatusRunning) && details.Status != string(models.ProjectStatusPartiallyRunning) {+ return}- // Log success event- _, _ = s.eventService.CreateEvent(syncCtx, CreateEventRequest{+ slog.InfoContext(ctx, "Redeploying project due to content change from Git sync", "syncMode", syncMode, "projectName", project.Name, "projectId", project.ID)+ if err := s.projectService.RedeployProject(ctx, project.ID, actor); err != nil {+ slog.ErrorContext(ctx, "Failed to redeploy project after Git sync", "syncMode", syncMode, "error", err, "projectId", project.ID)+ }+}++// logSyncSuccess records the Git sync completion event once the filesystem and+// sync-status updates have already succeeded.+func (s *GitOpsSyncService) logSyncSuccess(ctx context.Context, sync *models.GitOpsSync, project *models.Project, actor models.User) {+ _, _ = s.eventService.CreateEvent(ctx, CreateEventRequest{Type: models.EventTypeGitSyncRun,Severity: models.EventSeveritySuccess,Title: "Git sync completed",@@ -586,10 +631,6 @@ func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id sUsername: new(actor.Username),EnvironmentID: new(sync.EnvironmentID),})-- slog.InfoContext(syncCtx, "GitOps sync completed", "syncId", id, "project", project.Name)-- return result, nil}func (s *GitOpsSyncService) updateSyncStatus(ctx context.Context, id, status, errorMsg, commitHash string) {@@ -803,12 +844,6 @@ func (s *GitOpsSyncService) createProjectForSyncInternal(ctx context.Context, syslog.InfoContext(ctx, "Created project for GitOps sync", "projectName", sync.ProjectName, "projectId", project.ID)- // Deploy the project immediately after creation- slog.InfoContext(ctx, "Deploying project after initial Git sync", "projectName", project.Name, "projectId", project.ID)- if err := s.projectService.DeployProject(ctx, project.ID, actor, nil); err != nil {- slog.ErrorContext(ctx, "Failed to deploy project after initial Git sync", "error", err, "projectId", project.ID)- }-return project, nil}@@ -872,12 +907,6 @@ func envContentChangedInternal(oldEnv, newEnv string) bool {return !maps.Equal(oldEnvMap, newEnvMap)}-// getProjectsDirectory returns the configured projects directory path-func (s *GitOpsSyncService) getProjectsDirectory(ctx context.Context) (string, error) {- projectsDirSetting := s.settingsService.GetStringSetting(ctx, "projectsDirectory", "/app/data/projects")- return projects.GetProjectsDirectory(ctx, strings.TrimSpace(projectsDirSetting))-}-// parseSyncedFiles parses the JSON array of synced file paths from the databasefunc parseSyncedFiles(syncedFilesJSON *string) []string {if syncedFilesJSON == nil || *syncedFilesJSON == "" {@@ -946,36 +975,401 @@ func (s *GitOpsSyncService) walkAndParseSyncDirectory(ctx context.Context, syncreturn composeContent, syncFiles, nil}-// writeSyncFilesToProject writes the given sync files to the project directory.-// If oldSyncedFiles is provided, removed files will be cleaned up first.-func (s *GitOpsSyncService) writeSyncFilesToProject(ctx context.Context, sync *models.GitOpsSync, project *models.Project, syncFiles []projects.SyncFile, oldSyncedFiles []string) ([]string, error) {- projectsDir, err := s.getProjectsDirectory(ctx)+// syncProjectDirectoryInternal runs the new directory-sync path end to end:+// stage files, validate the staged tree, then create or update the project.+func (s *GitOpsSyncService) syncProjectDirectoryInternal(ctx context.Context, sync *models.GitOpsSync, syncFiles []projects.SyncFile, actor models.User) (*models.Project, []string, bool, bool, error) {+ stage, err := s.stageDirectorySyncInternal(ctx, sync, syncFiles)+ if err != nil {+ return nil, nil, false, false, err+ }+ defer func() {+ if stage != nil && stage.stagePath != "" {+ _ = os.RemoveAll(stage.stagePath)+ }+ }()++ if stage.project == nil {+ project, err := s.createDirectorySyncProjectInternal(ctx, sync, stage, actor)+ if err != nil {+ return nil, nil, false, false, err+ }+ return project, stage.syncedFiles, true, true, nil+ }++ project, err := s.updateDirectorySyncProjectInternal(ctx, sync, stage)+ if err != nil {+ return nil, nil, false, false, err+ }+ return project, stage.syncedFiles, false, stage.contentsChanged, nil+}++// stageDirectorySyncInternal builds a temporary project tree that reflects the exact+// repo layout after sync, including cleanup of files removed from the repo.+func (s *GitOpsSyncService) stageDirectorySyncInternal(ctx context.Context, sync *models.GitOpsSync, syncFiles []projects.SyncFile) (*stagedDirectorySync, error) {+ projectsDir, err := s.projectService.getProjectsDirectoryInternal(ctx)if err != nil {return nil, fmt.Errorf("failed to get projects directory: %w", err)}- // Build list of new file paths- newFiles := make([]string, len(syncFiles))- for i, f := range syncFiles {- newFiles[i] = f.RelativePath+ stagePath, err := os.MkdirTemp(projectsDir, ".gitops-sync-stage-*")+ if err != nil {+ return nil, fmt.Errorf("failed to create staging directory: %w", err)+ }++ project, err := s.getDirectorySyncProjectInternal(ctx, sync)+ if err != nil {+ _ = os.RemoveAll(stagePath)+ return nil, err+ }++ if project != nil {+ if err := copyDirectoryContentsInternal(project.Path, stagePath); err != nil {+ _ = os.RemoveAll(stagePath)+ return nil, fmt.Errorf("failed to stage current project files: %w", err)+ }}- // Clean up removed files if we have old sync data+ syncedFiles := make([]string, len(syncFiles))+ for i, file := range syncFiles {+ syncedFiles[i] = file.RelativePath+ }++ oldSyncedFiles := parseSyncedFiles(sync.SyncedFiles)if len(oldSyncedFiles) > 0 {- if err := projects.CleanupRemovedFiles(projectsDir, project.Path, oldSyncedFiles, newFiles); err != nil {- slog.WarnContext(ctx, "Failed to cleanup removed files", "error", err, "syncId", sync.ID)- // Continue despite cleanup error - it's best effort+ if err := projects.CleanupRemovedFiles(projectsDir, stagePath, oldSyncedFiles, syncedFiles); err != nil {+ _ = os.RemoveAll(stagePath)+ return nil, fmt.Errorf("failed to clean removed synced files: %w", err)}}- // Write all files to project directory- writtenPaths, err := projects.WriteSyncedDirectory(projectsDir, project.Path, syncFiles)+ composeFileName := filepath.Base(sync.ComposePath)+ if err := removeStaleComposeFilesInternal(stagePath, composeFileName, syncedFiles); err != nil {+ _ = os.RemoveAll(stagePath)+ return nil, fmt.Errorf("failed to remove stale compose files: %w", err)+ }++ contentsChanged, err := directorySyncContentsChangedInternal(project, syncFiles, oldSyncedFiles, composeFileName)+ if err != nil {+ _ = os.RemoveAll(stagePath)+ return nil, fmt.Errorf("failed to compare staged directory changes: %w", err)+ }++ // Write the repo files after cleanup so validation sees the final on-disk+ // tree exactly as it will exist in the managed project.+ if _, err := projects.WriteSyncedDirectory(projectsDir, stagePath, syncFiles); err != nil {+ _ = os.RemoveAll(stagePath)+ return nil, fmt.Errorf("failed to write staged sync files: %w", err)+ }++ serviceCount, err := s.validateDirectorySyncStageInternal(ctx, sync.ProjectName, stagePath, composeFileName)+ if err != nil {+ _ = os.RemoveAll(stagePath)+ return nil, fmt.Errorf("invalid compose file: %w", err)+ }++ return &stagedDirectorySync{+ stagePath: stagePath,+ composeFileName: composeFileName,+ project: project,+ syncedFiles: syncedFiles,+ serviceCount: serviceCount,+ contentsChanged: contentsChanged,+ }, nil+}++// getDirectorySyncProjectInternal resolves the linked project for a sync when one+// exists, while tolerating deleted/stale project references.+func (s *GitOpsSyncService) getDirectorySyncProjectInternal(ctx context.Context, sync *models.GitOpsSync) (*models.Project, error) {+ if sync.ProjectID == nil || *sync.ProjectID == "" {+ return nil, nil+ }++ project, err := s.projectService.GetProjectFromDatabaseByID(ctx, *sync.ProjectID)if err != nil {- return nil, fmt.Errorf("failed to write synced directory: %w", err)+ slog.WarnContext(ctx, "Existing project not found, will create new one", "projectId", *sync.ProjectID, "error", err)+ return nil, nil}- slog.InfoContext(ctx, "Directory sync written", "syncId", sync.ID, "projectId", project.ID, "filesWritten", len(writtenPaths))- return writtenPaths, nil+ if err := s.projectService.ensureProjectPathUnderRoot(ctx, project, false); err != nil {+ return nil, err+ }++ return project, nil+}++// validateDirectorySyncStageInternal loads the staged compose project using the real+// synced compose filename so include/env_file resolution happens against the+// fully copied directory contents.+func (s *GitOpsSyncService) validateDirectorySyncStageInternal(ctx context.Context, projectName, stagePath, composeFileName string) (int, error) {+ projectsDir, err := s.projectService.getProjectsDirectoryInternal(ctx)+ if err != nil {+ return 0, err+ }++ pathMapper, pmErr := s.projectService.getPathMapper(ctx)+ if pmErr != nil {+ slog.WarnContext(ctx, "failed to create path mapper for directory sync validation, continuing without translation", "error", pmErr)+ }++ autoInjectEnv := s.settingsService.GetBoolSetting(ctx, "autoInjectEnv", false)+ project, err := projects.LoadComposeProject(+ ctx,+ filepath.Join(stagePath, composeFileName),+ normalizeComposeProjectName(projectName),+ projectsDir,+ autoInjectEnv,+ pathMapper,+ )+ if err != nil {+ return 0, err+ }++ return len(project.Services), nil+}++// createDirectorySyncProjectInternal promotes a validated staged tree into a new+// managed project directory and links it back to the Git sync record.+func (s *GitOpsSyncService) createDirectorySyncProjectInternal(ctx context.Context, sync *models.GitOpsSync, stage *stagedDirectorySync, actor models.User) (*models.Project, error) {+ projectsDir, err := s.projectService.getProjectsDirectoryInternal(ctx)+ if err != nil {+ return nil, fmt.Errorf("failed to get projects directory: %w", err)+ }++ basePath := filepath.Join(projectsDir, projects.SanitizeProjectName(sync.ProjectName))+ projectPath, folderName, err := projects.CreateUniqueDir(projectsDir, basePath, sync.ProjectName, 0o755)+ if err != nil {+ return nil, fmt.Errorf("failed to create project directory: %w", err)+ }++ if err := os.Remove(projectPath); err != nil {+ return nil, fmt.Errorf("failed to prepare project directory: %w", err)+ }++ if err := os.Rename(stage.stagePath, projectPath); err != nil {+ return nil, fmt.Errorf("failed to promote staged project directory: %w", err)+ }+ stage.stagePath = ""++ project := &models.Project{+ Name: sync.ProjectName,+ DirName: new(folderName),+ Path: projectPath,+ Status: models.ProjectStatusStopped,+ ServiceCount: stage.serviceCount,+ RunningCount: 0,+ }++ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := tx.Create(project).Error; err != nil {+ return fmt.Errorf("failed to create project: %w", err)+ }++ if err := tx.Model(&models.GitOpsSync{}).Where("id = ?", sync.ID).Update("project_id", project.ID).Error; err != nil {+ return fmt.Errorf("failed to update sync with project ID: %w", err)+ }++ if err := tx.Model(&models.Project{}).Where("id = ?", project.ID).Update("gitops_managed_by", sync.ID).Error; err != nil {+ return fmt.Errorf("failed to mark project as GitOps-managed: %w", err)+ }++ return nil+ }); err != nil {+ _ = os.RemoveAll(projectPath)+ return nil, err+ }++ sync.ProjectID = &project.ID+ s.projectService.cacheComposeProjectIDInternal(normalizeComposeProjectName(project.Name), project.ID)++ if s.projectService.eventService != nil {+ metadata := models.JSON{"action": "create", "projectID": project.ID, "projectName": project.Name, "path": projectPath}+ if logErr := s.projectService.eventService.LogProjectEvent(ctx, models.EventTypeProjectCreate, project.ID, project.Name, actor.ID, actor.Username, "0", metadata); logErr != nil {+ slog.ErrorContext(ctx, "could not log project creation", "error", logErr)+ }+ }++ return project, nil+}++// updateDirectorySyncProjectInternal swaps a validated staged tree into the existing+// project path with a temporary backup so failed promotion can roll back.+func (s *GitOpsSyncService) updateDirectorySyncProjectInternal(ctx context.Context, sync *models.GitOpsSync, stage *stagedDirectorySync) (*models.Project, error) {+ project := stage.project+ projectPath := filepath.Clean(project.Path)+ backupPath := ""++ if info, err := os.Stat(projectPath); err == nil {+ if !info.IsDir() {+ return nil, fmt.Errorf("project path is not a directory: %s", projectPath)+ }+ backupPath = fmt.Sprintf("%s.gitops-backup-%d", projectPath, time.Now().UnixNano())+ if err := os.Rename(projectPath, backupPath); err != nil {+ return nil, fmt.Errorf("failed to move current project directory out of the way: %w", err)+ }+ } else if !errors.Is(err, os.ErrNotExist) {+ return nil, fmt.Errorf("failed to inspect current project directory: %w", err)+ }++ if err := os.Rename(stage.stagePath, projectPath); err != nil {+ if backupPath != "" {+ _ = os.Rename(backupPath, projectPath)+ }+ return nil, fmt.Errorf("failed to promote staged project directory: %w", err)+ }+ stage.stagePath = ""++ if err := s.db.WithContext(ctx).Model(&models.Project{}).Where("id = ?", project.ID).Updates(map[string]any{+ "service_count": stage.serviceCount,+ "gitops_managed_by": sync.ID,+ "updated_at": time.Now(),+ }).Error; err != nil {+ if backupPath != "" {+ _ = os.RemoveAll(projectPath)+ _ = os.Rename(backupPath, projectPath)+ }+ return nil, fmt.Errorf("failed to update project metadata after directory sync: %w", err)+ }++ if backupPath != "" {+ _ = os.RemoveAll(backupPath)+ }++ return project, nil+}++// directorySyncContentsChangedInternal compares the currently managed synced files with+// the newly fetched repo files to decide whether a redeploy is needed.+func directorySyncContentsChangedInternal(project *models.Project, syncFiles []projects.SyncFile, oldSyncedFiles []string, composeFileName string) (bool, error) {+ if project == nil {+ return true, nil+ }++ if info, err := os.Stat(project.Path); err != nil {+ if errors.Is(err, os.ErrNotExist) {+ return true, nil+ }+ return false, err+ } else if !info.IsDir() {+ return false, fmt.Errorf("project path is not a directory: %s", project.Path)+ }++ newFileSet := make(map[string]struct{}, len(syncFiles))+ for _, file := range syncFiles {+ newFileSet[file.RelativePath] = struct{}{}+ existingContent, err := os.ReadFile(filepath.Join(project.Path, file.RelativePath))+ if err != nil {+ if errors.Is(err, os.ErrNotExist) {+ return true, nil+ }+ return false, err+ }+ if !bytes.Equal(existingContent, file.Content) {+ return true, nil+ }+ }++ for _, oldFile := range oldSyncedFiles {+ if _, exists := newFileSet[oldFile]; exists {+ continue+ }+ if _, err := os.Stat(filepath.Join(project.Path, oldFile)); err == nil {+ return true, nil+ } else if !errors.Is(err, os.ErrNotExist) {+ return false, err+ }+ }++ for _, candidate := range projects.ComposeFileCandidates() {+ if candidate == composeFileName {+ continue+ }+ if _, exists := newFileSet[candidate]; exists {+ continue+ }+ if _, err := os.Stat(filepath.Join(project.Path, candidate)); err == nil {+ return true, nil+ } else if !errors.Is(err, os.ErrNotExist) {+ return false, err+ }+ }++ return false, nil+}++// removeStaleComposeFilesInternal drops old canonical compose filenames from the staged+// project when the repo now uses a different compose filename.+func removeStaleComposeFilesInternal(projectPath, composeFileName string, syncedFiles []string) error {+ syncedFileSet := make(map[string]struct{}, len(syncedFiles))+ for _, file := range syncedFiles {+ syncedFileSet[file] = struct{}{}+ }++ for _, candidate := range projects.ComposeFileCandidates() {+ if candidate == composeFileName {+ continue+ }+ if _, exists := syncedFileSet[candidate]; exists {+ continue+ }+ if err := os.Remove(filepath.Join(projectPath, candidate)); err != nil && !errors.Is(err, os.ErrNotExist) {+ return err+ }+ }++ return nil+}++// copyDirectoryContentsInternal clones the current project tree into the staging+// directory so directory sync updates can preserve non-synced local files.+func copyDirectoryContentsInternal(srcDir, destDir string) error {+ srcRoot, err := os.OpenRoot(srcDir)+ if err != nil {+ return err+ }+ defer func() { _ = srcRoot.Close() }()++ destRoot, err := os.OpenRoot(destDir)+ if err != nil {+ return err+ }+ defer func() { _ = destRoot.Close() }()++ return filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error {+ if walkErr != nil {+ return walkErr+ }+ if path == srcDir {+ return nil+ }++ relPath, err := filepath.Rel(srcDir, path)+ if err != nil {+ return err+ }++ if d.IsDir() {+ return destRoot.MkdirAll(relPath, 0o755)+ }+ if d.Type()&os.ModeSymlink != 0 {+ return nil+ }++ info, err := d.Info()+ if err != nil {+ return err+ }++ content, err := srcRoot.ReadFile(relPath)+ if err != nil {+ return err+ }++ if err := destRoot.MkdirAll(filepath.Dir(relPath), 0o755); err != nil {+ return err+ }++ return destRoot.WriteFile(relPath, content, info.Mode())+ })}// updateSyncStatusWithFiles updates sync status including the list of synced filesdiff --git a/backend/internal/services/gitops_sync_service_test.go b/backend/internal/services/gitops_sync_service_test.goindex a1939e03a..8f9b15151 100644--- a/backend/internal/services/gitops_sync_service_test.go+++ b/backend/internal/services/gitops_sync_service_test.go@@ -2,13 +2,246 @@ package servicesimport ("context"+ "encoding/json"+ "errors"+ "os"+ "path/filepath""testing"+ "github.com/getarcaneapp/arcane/backend/internal/database""github.com/getarcaneapp/arcane/backend/internal/models"+ "github.com/getarcaneapp/arcane/backend/pkg/projects""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "gorm.io/gorm")+func setupGitOpsSyncDirectoryTestService(t *testing.T) (*GitOpsSyncService, *database.DB, string) {+ t.Helper()++ ctx := context.Background()+ db := setupProjectTestDB(t)+ require.NoError(t, db.AutoMigrate(&models.GitOpsSync{}))++ settingsService, err := NewSettingsService(ctx, db)+ require.NoError(t, err)++ projectsDir := t.TempDir()+ require.NoError(t, settingsService.SetStringSetting(ctx, "projectsDirectory", projectsDir))++ projectService := NewProjectService(db, settingsService, nil, nil, nil, nil)++ return NewGitOpsSyncService(db, nil, projectService, nil, settingsService), db, projectsDir+}++func TestGitOpsSyncService_SyncProjectDirectory_CreatesProjectPreservingRepoLayout(t *testing.T) {+ ctx := context.Background()+ svc, db, _ := setupGitOpsSyncDirectoryTestService(t)++ sync := &models.GitOpsSync{+ BaseModel: models.BaseModel{ID: "sync-directory-create"},+ Name: "demo-sync",+ EnvironmentID: "0",+ RepositoryID: "repo-1",+ ComposePath: "apps/demo/docker-compose.yaml",+ ProjectName: "demo-project",+ SyncDirectory: true,+ }+ require.NoError(t, db.Create(sync).Error)++ syncFiles := []projects.SyncFile{+ {+ RelativePath: "docker-compose.yaml",+ Content: []byte(`include:+ - meta.yaml+services:+ app:+ image: nginx:alpine+ env_file:+ - .env+`),+ },+ {+ RelativePath: "meta.yaml",+ Content: []byte(`services:+ helper:+ image: busybox:latest+`),+ },+ {+ RelativePath: ".env",+ Content: []byte("APP_MODE=production\n"),+ },+ }++ project, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, models.User{})+ require.NoError(t, err)+ require.NotNil(t, project)+ require.True(t, created)+ require.True(t, changed)+ require.ElementsMatch(t, []string{"docker-compose.yaml", "meta.yaml", ".env"}, syncedFiles)++ composePath, detectErr := projects.DetectComposeFile(project.Path)+ require.NoError(t, detectErr)+ assert.Equal(t, filepath.Join(project.Path, "docker-compose.yaml"), composePath)++ composeBytes, err := os.ReadFile(filepath.Join(project.Path, "docker-compose.yaml"))+ require.NoError(t, err)+ assert.Contains(t, string(composeBytes), "include:")++ metaBytes, err := os.ReadFile(filepath.Join(project.Path, "meta.yaml"))+ require.NoError(t, err)+ assert.Contains(t, string(metaBytes), "helper:")++ envBytes, err := os.ReadFile(filepath.Join(project.Path, ".env"))+ require.NoError(t, err)+ assert.Equal(t, "APP_MODE=production\n", string(envBytes))++ _, statErr := os.Stat(filepath.Join(project.Path, "compose.yaml"))+ assert.ErrorIs(t, statErr, os.ErrNotExist)+}++func TestGitOpsSyncService_SyncProjectDirectory_UpdatesProjectAndCleansOldSyncedFiles(t *testing.T) {+ ctx := context.Background()+ svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)++ projectPath := filepath.Join(projectsDir, "demo-project")+ require.NoError(t, os.MkdirAll(filepath.Join(projectPath, "nested"), 0o755))+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "docker-compose.yaml"), []byte(`include:+ - meta.yaml+services:+ app:+ image: nginx:1.26-alpine+`), 0o644))+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "meta.yaml"), []byte(`services:+ helper:+ image: busybox:1.36+`), 0o644))+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "old.txt"), []byte("remove me\n"), 0o644))+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "keep.txt"), []byte("keep me\n"), 0o644))+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services: {}\n"), 0o644))++ project := &models.Project{+ BaseModel: models.BaseModel{ID: "proj-directory-update"},+ Name: "demo-project",+ DirName: new("demo-project"),+ Path: projectPath,+ Status: models.ProjectStatusStopped,+ }+ require.NoError(t, db.Create(project).Error)++ oldSyncedFilesJSON, err := json.Marshal([]string{"docker-compose.yaml", "meta.yaml", "old.txt"})+ require.NoError(t, err)++ sync := &models.GitOpsSync{+ BaseModel: models.BaseModel{ID: "sync-directory-update"},+ Name: "demo-sync",+ EnvironmentID: "0",+ RepositoryID: "repo-1",+ ComposePath: "apps/demo/docker-compose.yaml",+ ProjectName: "demo-project",+ ProjectID: &project.ID,+ SyncDirectory: true,+ SyncedFiles: new(string(oldSyncedFilesJSON)),+ }+ require.NoError(t, db.Create(sync).Error)++ syncFiles := []projects.SyncFile{+ {+ RelativePath: "docker-compose.yaml",+ Content: []byte(`include:+ - nested/feature.yaml+services:+ app:+ image: nginx:1.27-alpine+`),+ },+ {+ RelativePath: "nested/feature.yaml",+ Content: []byte(`services:+ worker:+ image: busybox:latest+`),+ },+ }++ updatedProject, syncedFiles, created, changed, err := svc.syncProjectDirectoryInternal(ctx, sync, syncFiles, models.User{})+ require.NoError(t, err)+ require.NotNil(t, updatedProject)+ require.False(t, created)+ require.True(t, changed)+ require.ElementsMatch(t, []string{"docker-compose.yaml", "nested/feature.yaml"}, syncedFiles)++ composePath, detectErr := projects.DetectComposeFile(updatedProject.Path)+ require.NoError(t, detectErr)+ assert.Equal(t, filepath.Join(updatedProject.Path, "docker-compose.yaml"), composePath)++ _, statErr := os.Stat(filepath.Join(updatedProject.Path, "old.txt"))+ assert.ErrorIs(t, statErr, os.ErrNotExist)++ _, statErr = os.Stat(filepath.Join(updatedProject.Path, "compose.yaml"))+ assert.ErrorIs(t, statErr, os.ErrNotExist)++ keepBytes, err := os.ReadFile(filepath.Join(updatedProject.Path, "keep.txt"))+ require.NoError(t, err)+ assert.Equal(t, "keep me\n", string(keepBytes))++ featureBytes, err := os.ReadFile(filepath.Join(updatedProject.Path, "nested", "feature.yaml"))+ require.NoError(t, err)+ assert.Contains(t, string(featureBytes), "worker:")+}++func TestGitOpsSyncService_CreateDirectorySyncProjectInternal_RollsBackProjectOnUpdateFailure(t *testing.T) {+ ctx := context.Background()+ svc, db, projectsDir := setupGitOpsSyncDirectoryTestService(t)++ sync := &models.GitOpsSync{+ BaseModel: models.BaseModel{ID: "sync-directory-tx-rollback"},+ Name: "demo-sync",+ EnvironmentID: "0",+ RepositoryID: "repo-1",+ ComposePath: "apps/demo/docker-compose.yaml",+ ProjectName: "demo-project",+ SyncDirectory: true,+ }+ require.NoError(t, db.Create(sync).Error)++ stagePath := t.TempDir()+ require.NoError(t, os.WriteFile(filepath.Join(stagePath, "docker-compose.yaml"), []byte("services: {}\n"), 0o644))++ stage := &stagedDirectorySync{+ stagePath: stagePath,+ composeFileName: "docker-compose.yaml",+ serviceCount: 1,+ }++ callbackName := "test:fail_project_gitops_update"+ require.NoError(t, db.Callback().Update().Before("gorm:update").Register(callbackName, func(tx *gorm.DB) {+ if tx.Statement != nil && tx.Statement.Table == "projects" {+ tx.AddError(errors.New("forced project update failure"))+ }+ }))+ defer func() {+ db.Callback().Update().Remove(callbackName)+ }()++ project, err := svc.createDirectorySyncProjectInternal(ctx, sync, stage, models.User{})+ require.Error(t, err)+ require.Nil(t, project)+ assert.Contains(t, err.Error(), "failed to mark project as GitOps-managed")++ var projectCount int64+ require.NoError(t, db.Model(&models.Project{}).Count(&projectCount).Error)+ assert.Zero(t, projectCount)++ var storedSync models.GitOpsSync+ require.NoError(t, db.First(&storedSync, "id = ?", sync.ID).Error)+ assert.Nil(t, storedSync.ProjectID)++ _, statErr := os.Stat(filepath.Join(projectsDir, "demo-project"))+ assert.ErrorIs(t, statErr, os.ErrNotExist)+}+func TestEnvContentChangedInternal(t *testing.T) {t.Run("ignores formatting-only changes", func(t *testing.T) {oldEnv := "B=2\nA=1\n# comment\n"diff --git a/backend/pkg/projects/fs_writer.go b/backend/pkg/projects/fs_writer.goindex 2dc724df8..c8e66aa43 100644--- a/backend/pkg/projects/fs_writer.go+++ b/backend/pkg/projects/fs_writer.go@@ -20,6 +20,12 @@ var composeFileCandidates = []string{"podman-compose.yml",}+// ComposeFileCandidates returns the supported compose filenames in Arcane's+// detection order. A copy is returned so callers can't mutate package state.+func ComposeFileCandidates() []string {+ return append([]string(nil), composeFileCandidates...)+}+// detectExistingComposeFile finds an existing compose file in the directoryfunc detectExistingComposeFile(dir string) string {for _, filename := range composeFileCandidates {