Skip to content

Modern Docker Management, Designed for Everyone

48.4 KB
Raw Blame
package gitops

import (
	"cmp"
	"context"
	"errors"
	"fmt"
	"io/fs"
	"log/slog"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/getarcaneapp/arcane/types/v2/base"
	"github.com/getarcaneapp/arcane/types/v2/gitops"
	"github.com/getarcaneapp/arcane/types/v2/lifecycle"
	"github.com/getarcaneapp/arcane/types/v2/scheduler"
	"github.com/getarcaneapp/arcane/types/v2/user"
	"github.com/italypaleale/francis/builtin/workflow"
	"go.getarcane.app/acfs"
	"go.getarcane.app/kit/pkg"
	"go.getarcane.app/kit/pkg/mapping"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/metric"
	"go.opentelemetry.io/otel/trace"
	"gorm.io/gorm"
	"gorm.io/gorm/clause"

	"github.com/getarcaneapp/arcane/backend/v2/internal/common"
	"github.com/getarcaneapp/arcane/backend/v2/internal/database"
	"github.com/getarcaneapp/arcane/backend/v2/internal/event"
	"github.com/getarcaneapp/arcane/backend/v2/internal/gitops/children/backup"
	gitopssync "github.com/getarcaneapp/arcane/backend/v2/internal/gitops/children/sync"
	"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
	projectpkg "github.com/getarcaneapp/arcane/backend/v2/internal/project"
	"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
	"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/gitutil"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/projects"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/entityjobs"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)

var syncRuns = func() metric.Int64Counter {
	counter, err := otel.Meter(tracing.InstrumentationName).Int64Counter("arcane.gitops.syncs",
		metric.WithDescription("GitOps sync runs by outcome"), metric.WithUnit("{sync}"))
	if err != nil {
		otel.Handle(err)
	}
	return counter
}()

const (
	defaultGitSyncTimeout = 5 * time.Minute

	defaultMaxSyncFiles        = 500
	defaultMaxSyncTotalSizeMB  = 50
	defaultMaxSyncBinarySizeMB = 10

	gitOpsSyncAdmissionScope = "gitops-sync"
)

type (
	GitOpsSyncService struct {
		db              *database.DB
		repoService     *gitrepo.GitRepositoryService
		projectService  *projectpkg.ProjectService
		swarmService    *swarm.SwarmService
		eventService    *event.EventService
		settingsService *settings.SettingsService

		// jobs carries the scheduler and app lifecycle context, injected after construction by SetScheduler.
		jobs         *entityjobs.Registry
		engine       *flow.Engine
		syncWorkflow *flow.Workflow

		backups *backupRuntime
		sync    *gitopssync.Service
		backup  *backup.Service
	}

	// syncTarget is the gitops-sync workflow input: the sync one scheduled job runs.
	syncTarget struct {
		EnvironmentID string `json:"environmentId"`
		SyncID        string `json:"syncId"`
	}

	// lifecycleConfigInput is the pre-deploy hook slice of a create or update request.
	lifecycleConfigInput struct {
		targetType    *string
		scriptPath    *string
		runnerImage   *string
		env           *string
		extraMounts   *string
		timeoutSec    *int
		networkMode   *string
		syncDirectory *bool
	}

	// backupRuntime holds the in-memory coordination for backup syncs.
	backupRuntime struct {
		mu       sync.Mutex
		branches map[string]*sync.Mutex
		timers   map[string]*time.Timer
		debounce time.Duration
	}
)

func (s *GitOpsSyncService) validateLifecycleConfig(ctx context.Context, current *projectpkg.GitOpsSync, in lifecycleConfigInput) error {
	var currentScript, currentRunner *string
	currentTargetType, currentSyncDirectory := "", false
	if current != nil {
		currentScript, currentRunner = current.PreDeployScriptPath, current.PreDeployRunnerImage
		currentTargetType, currentSyncDirectory = strings.TrimSpace(current.TargetType), current.SyncDirectory
	}
	currentTargetType = cmp.Or(currentTargetType, "project")
	targetType := cmp.Or(strings.TrimSpace(kit.FromPtr(in.targetType)), currentTargetType)
	scriptPath := strings.TrimSpace(kit.FromPtr(cmp.Or(in.scriptPath, currentScript)))
	invalid := func(field, message string) error {
		return common.Classify(common.ErrValidation, &base.FieldError{Field: field, Err: errors.New(message)})
	}

	lifecycleFieldSet := in.scriptPath != nil || in.runnerImage != nil || in.env != nil || in.extraMounts != nil || in.timeoutSec != nil || in.networkMode != nil
	targetTypeChanging := in.targetType != nil && strings.TrimSpace(*in.targetType) != currentTargetType
	// Nothing lifecycle-related changes, or the result has no script to validate.
	if !lifecycleFieldSet && (scriptPath == "" || (in.syncDirectory == nil && !targetTypeChanging)) {
		return nil
	}
	// The kill switch only blocks edits to lifecycle fields, not unrelated edits to a sync that already has a hook.
	if lifecycleFieldSet && !s.settingsService.GetBoolSetting(ctx, "lifecycleEnabled", false) {
		return invalid("preDeployScriptPath", "Pre-deploy lifecycle hooks are disabled. An admin must enable lifecycleEnabled in settings before they can be configured.")
	}

	// scriptPath is a POSIX repo path, so path.IsAbs behaves the same on Windows.
	cleanedScript := filepath.ToSlash(filepath.Clean(scriptPath))
	switch {
	case scriptPath == "":
	case targetType == "swarm_stack":
		return invalid("preDeployScriptPath", "Pre-deploy lifecycle hooks are only supported for project syncs.")
	case len(scriptPath) > 256:
		return invalid("preDeployScriptPath", "Script path must be 256 characters or fewer.")
	case path.IsAbs(filepath.ToSlash(scriptPath)):
		return invalid("preDeployScriptPath", "Script path must be relative to the project directory.")
	case cleanedScript == ".." || strings.HasPrefix(cleanedScript, "../"):
		return invalid("preDeployScriptPath", "Script path must not escape the project directory.")
	case strings.TrimSpace(kit.FromPtr(cmp.Or(in.runnerImage, currentRunner))) == "" && strings.TrimSpace(s.settingsService.GetStringSetting(ctx, "lifecycleDefaultRunnerImage", "")) == "":
		return invalid("preDeployRunnerImage", "Runner image is required when a script path is set.")
	case !kit.FromPtr(cmp.Or(in.syncDirectory, &currentSyncDirectory)):
		return invalid("preDeployScriptPath", "Pre-deploy script requires \"Sync entire directory\" so the script is included in the synced files.")
	}

	if in.timeoutSec != nil {
		maxTimeoutSec := s.settingsService.GetIntSetting(ctx, "lifecycleMaxTimeoutSec", lifecycle.DefaultMaxTimeoutSec)
		if *in.timeoutSec < 1 {
			return invalid("preDeployTimeoutSec", "Timeout must be at least 1 second.")
		}
		if maxTimeoutSec > 0 && *in.timeoutSec > maxTimeoutSec {
			return invalid("preDeployTimeoutSec", fmt.Sprintf("Timeout %ds exceeds the lifecycleMaxTimeoutSec setting (%ds).", *in.timeoutSec, maxTimeoutSec))
		}
	}
	if _, err := projectpkg.ParseEnvText(in.env); err != nil {
		return invalid("preDeployEnv", err.Error())
	}
	if _, err := projectpkg.ParseExtraMountsText(in.extraMounts); err != nil {
		return invalid("preDeployExtraMounts", err.Error())
	}
	return nil
}

func NewGitOpsSyncService(
	db *database.DB,
	repoService *gitrepo.GitRepositoryService,
	projectService *projectpkg.ProjectService,
	swarmService *swarm.SwarmService,
	eventService *event.EventService,
	settingsService *settings.SettingsService,
) *GitOpsSyncService {
	s := &GitOpsSyncService{
		db:              db,
		repoService:     repoService,
		projectService:  projectService,
		swarmService:    swarmService,
		eventService:    eventService,
		settingsService: settingsService,
		jobs:            entityjobs.New(entityjobs.GitOpsSyncJobPrefix, gitOpsSyncAdmissionScope),
		backups:         &backupRuntime{branches: map[string]*sync.Mutex{}, timers: map[string]*time.Timer{}, debounce: backup.DefaultBackupSaveDebounce},
	}
	logError := func(ctx context.Context, syncRecord *projectpkg.GitOpsSync, actor user.Actor, errorMsg string) {
		_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
			Type:          event.EventTypeGitSyncError,
			Severity:      event.EventSeverityError,
			Title:         "Git sync failed",
			Description:   fmt.Sprintf("Failed to sync '%s': %s", syncRecord.Name, errorMsg),
			ResourceType:  new("git_sync"),
			ResourceID:    new(syncRecord.ID),
			ResourceName:  new(syncRecord.Name),
			UserID:        new(actor.ID),
			Username:      new(actor.Username),
			EnvironmentID: new(syncRecord.EnvironmentID),
		})
	}
	s.sync = gitopssync.New(db, repoService, projectService, swarmService, eventService, s.getEffectiveSyncLimits, logError, s.jobs.Unregister)
	s.backup = backup.New(db, repoService, projectService, eventService, s.getEffectiveSyncLimits, logError, s.backups.branchLock)
	return s
}

// RegisterWorkflows defines the gitops-sync workflow each scheduled sync runs. A sync step retries transient
// clone failures; a retry replays the run's pinned revision.
func (s *GitOpsSyncService) RegisterWorkflows(engine *flow.Engine) error {
	s.engine = engine
	syncWorkflow, err := engine.Define(flow.Definition{
		Name:        "gitops-sync",
		Version:     1,
		Fingerprint: "f693aa29ca10c373e1b9d39ab25dee6e396ced7cea2bcfef76f41dba571d576a",
		Concurrency: 16,
		Timeout:     time.Hour,
		Steps: []workflow.StepSpec{
			workflow.Step("sync", engine.Handler(func(ctx context.Context, t flow.Task) (any, error) {
				var target syncTarget
				if err := t.Payload(&target); err != nil {
					return nil, err
				}
				// The row is re-read on every run, so a deleted sync unregisters itself and a disabled one skips.
				syncRecord, err := s.getSyncByID(ctx, target.EnvironmentID, target.SyncID, false)
				if errors.Is(err, common.ErrNotFound) {
					slog.InfoContext(ctx, "gitops auto-sync job unregistering; sync no longer exists", "syncId", target.SyncID)
					s.jobs.Unregister(ctx, target.SyncID)
					return scheduler.Outcome{Status: scheduler.Skipped}, nil
				}
				if err != nil {
					return nil, common.Classify(common.ErrUnavailable, err)
				}
				if !syncRecord.AutoSync {
					return scheduler.Outcome{Status: scheduler.Skipped}, nil
				}
				if previous, ok := jobcontext.Run(ctx); ok {
					if outcome := jobcontext.ConfirmedTarget(previous, target.SyncID); outcome.Status == scheduler.Succeeded {
						return outcome, nil
					}
				}
				if runningErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: target.SyncID, Status: scheduler.Running}); runningErr != nil {
					return nil, runningErr
				}
				result, err := s.PerformSync(ctx, target.EnvironmentID, target.SyncID, user.SystemUser)
				if err != nil {
					slog.ErrorContext(ctx, "gitops auto-sync run failed", "syncId", target.SyncID, "error", err)
					return nil, err
				}
				if !result.Success {
					return scheduler.Outcome{Status: scheduler.Skipped, Message: result.Message}, nil
				}
				if succeededErr := jobcontext.Progress(ctx, scheduler.TargetOutcome{ID: target.SyncID, Status: scheduler.Succeeded}); succeededErr != nil {
					return nil, succeededErr
				}
				return scheduler.Outcome{Status: scheduler.Succeeded, Message: result.Message}, nil
			}), workflow.WithMaxAttempts(3), workflow.WithRetryBackoff(30*time.Second, 2*time.Minute)),
		},
	})
	s.syncWorkflow = syncWorkflow
	return err
}

// SetScheduler injects the scheduler, the admission gate, and the app lifecycle context background syncs run on.
// It must run during bootstrap, before any per-sync job is registered.
func (s *GitOpsSyncService) SetScheduler(ctx context.Context, jobScheduler scheduler.DynamicScheduler, admissionGate *runs.Admission) error {
	return s.jobs.SetScheduler(ctx, jobScheduler, admissionGate)
}

// registerSyncJob schedules one sync every intervalMinutes as a gitops-sync workflow run.
func (s *GitOpsSyncService) registerSyncJob(ctx context.Context, syncID, environmentID string, intervalMinutes int) {
	schedule := fmt.Sprintf("@every %dm", max(intervalMinutes, 1))
	s.jobs.Add(ctx, &flow.Job{
		Engine:     s.engine,
		Workflow:   s.syncWorkflow,
		JobName:    s.jobs.JobName(syncID),
		Payload:    syncTarget{EnvironmentID: environmentID, SyncID: syncID},
		ScheduleFn: func(context.Context) string { return schedule },
	})
}

// kickSync submits one run of a sync's job now, so it does not wait for its next interval.
func (s *GitOpsSyncService) kickSync(ctx context.Context, syncID, trigger string) {
	if !s.jobs.Enabled() {
		return
	}
	if _, err := s.jobs.Scheduler().Submit(ctx, scheduler.Request{JobID: s.jobs.JobName(syncID), EnvironmentID: "0", Trigger: trigger}); err != nil {
		slog.ErrorContext(ctx, "gitops sync admission failed", "syncId", syncID, "trigger", trigger, "error", err)
	}
}

// RegisterAutoSyncJobsOnStartup registers a job for every auto-sync and kicks the overdue ones, and backups
// with pending changes, right away.
func (s *GitOpsSyncService) RegisterAutoSyncJobsOnStartup(ctx context.Context) {
	if !s.jobs.Enabled() {
		return
	}
	var syncs []projectpkg.GitOpsSync
	if err := s.db.WithContext(ctx).
		Where("auto_sync = ? AND environment_id IN (SELECT id FROM environments)", true).
		Find(&syncs).Error; err != nil {
		slog.ErrorContext(ctx, "Failed to load auto-sync jobs on startup", "error", err)
		return
	}
	for _, syncRecord := range syncs {
		s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
		overdue := syncRecord.LastSyncAt == nil || time.Since(*syncRecord.LastSyncAt) > time.Duration(max(syncRecord.SyncInterval, 1))*time.Minute
		if overdue || (syncRecord.Mode == gitops.SyncModeBackup && syncRecord.BackupPending) {
			s.kickSync(ctx, syncRecord.ID, "startup")
		}
	}
	slog.InfoContext(ctx, "Registered gitops auto-sync jobs on startup", "count", len(syncs))
}

func (s *GitOpsSyncService) getEnvironmentSyncLimits(ctx context.Context) (int, int64, int64) {
	cfg := s.settingsService.GetSettingsOrDefaults(ctx)
	maxFiles := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxFiles.Value, defaultMaxSyncFiles, strconv.Atoi), defaultMaxSyncFiles)
	maxTotalSizeMB := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxTotalSizeMb.Value, defaultMaxSyncTotalSizeMB, strconv.Atoi), defaultMaxSyncTotalSizeMB)
	maxBinarySizeMB := normalizeSyncLimitSetting(kit.ParseOrDefault(cfg.GitSyncMaxBinarySizeMb.Value, defaultMaxSyncBinarySizeMB, strconv.Atoi), defaultMaxSyncBinarySizeMB)
	return maxFiles, int64(maxTotalSizeMB) << 20, int64(maxBinarySizeMB) << 20
}

// getEffectiveSyncLimits returns the sync's own limits, except where an environment override pins the setting.
func (s *GitOpsSyncService) getEffectiveSyncLimits(ctx context.Context, syncRecord *projectpkg.GitOpsSync) (int, int64, int64) {
	maxFiles, maxTotalSize, maxBinarySize := s.getEnvironmentSyncLimits(ctx)
	if syncRecord == nil {
		return maxFiles, maxTotalSize, maxBinarySize
	}
	return kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxFiles"), maxFiles, syncRecord.MaxSyncFiles),
		kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxTotalSizeMb"), maxTotalSize, syncRecord.MaxSyncTotalSize),
		kit.Ternary(s.settingsService.IsEnvOverrideActive("gitSyncMaxBinarySizeMb"), maxBinarySize, syncRecord.MaxSyncBinarySize)
}

func (s *GitOpsSyncService) GetSyncsPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]gitops.GitOpsSync, pagination.Response, gitops.SyncCounts, error) {
	var syncs []projectpkg.GitOpsSync
	q := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).
		Where("environment_id = ?", environmentID)

	if term := strings.TrimSpace(params.Search); term != "" {
		searchPattern := "%" + term + "%"
		q = q.Where(
			"name LIKE ? OR branch LIKE ? OR compose_path LIKE ? OR backup_directory LIKE ?",
			searchPattern, searchPattern, searchPattern, searchPattern,
		)
	}

	q = pagination.ApplyBooleanFilter(q, "auto_sync", params.Filters["autoSync"])
	q = pagination.ApplyFilter(q, "mode", params.Filters["mode"])

	q = pagination.ApplyFilter(q, "repository_id", params.Filters["repositoryId"])
	q = pagination.ApplyFilter(q, "project_id", params.Filters["projectId"])

	var total, active, successful, backups int64
	if err := errors.Join(
		q.Session(&gorm.Session{}).Count(&total).Error,
		q.Session(&gorm.Session{}).Where("auto_sync = ?", true).Count(&active).Error,
		q.Session(&gorm.Session{}).Where("last_sync_status = ?", "success").Count(&successful).Error,
		q.Session(&gorm.Session{}).Where("mode = ?", gitops.SyncModeBackup).Count(&backups).Error,
	); err != nil {
		return nil, pagination.Response{}, gitops.SyncCounts{}, fmt.Errorf("failed to get sync counts: %w", err)
	}
	counts := gitops.SyncCounts{
		TotalSyncs:      int(total),
		ActiveSyncs:     int(active),
		SuccessfulSyncs: int(successful),
		DeploySyncs:     int(total - backups),
		BackupSyncs:     int(backups),
	}

	paginationResp, err := pagination.PaginateAndSortDB(params, q.Preload("Repository").Preload("Project"), &syncs)
	if err != nil {
		return nil, pagination.Response{}, gitops.SyncCounts{}, fmt.Errorf("failed to paginate gitops syncs: %w", err)
	}

	out, mapErr := mapping.MapSlice[projectpkg.GitOpsSync, gitops.GitOpsSync](syncs)
	if mapErr != nil {
		return nil, pagination.Response{}, gitops.SyncCounts{}, fmt.Errorf("failed to map syncs: %w", mapErr)
	}

	return out, paginationResp, counts, nil
}

func (s *GitOpsSyncService) GetSyncByID(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {
	syncRecord, err := s.getSyncByID(ctx, environmentID, id, true)
	if err != nil {
		if errors.Is(err, common.ErrNotFound) {
			slog.WarnContext(ctx, "GitOps sync not found", "syncId", id, "environmentId", environmentID)
			return nil, err
		}
		slog.ErrorContext(ctx, "Failed to get GitOps sync", "syncId", id, "environmentId", environmentID, "error", err)
		return nil, err
	}
	return syncRecord, nil
}

func (s *GitOpsSyncService) getSyncByID(ctx context.Context, environmentID, id string, preloadAssociations bool) (*projectpkg.GitOpsSync, error) {
	var syncRecord projectpkg.GitOpsSync
	q := s.db.WithContext(ctx).Where("id = ?", id)
	if preloadAssociations {
		q = q.Preload("Repository").Preload("Project")
	}
	if environmentID != "" {
		q = q.Where("environment_id = ?", environmentID)
	}
	if err := q.First(&syncRecord).Error; err != nil {
		if errors.Is(err, gorm.ErrRecordNotFound) {
			return nil, common.Classify(common.ErrNotFound, errors.New("GitOps sync not found"))
		}
		return nil, fmt.Errorf("failed to get sync: %w", err)
	}
	return &syncRecord, nil
}

func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string, req gitops.CreateSyncRequest, actor user.Actor) (*projectpkg.GitOpsSync, error) {
	slog.InfoContext(ctx, "Creating GitOps sync", "environmentId", environmentID, "name", req.Name, "repositoryId", req.RepositoryID)

	mode := cmp.Or(strings.TrimSpace(req.Mode), gitops.SyncModeDeploy)
	switch {
	case mode != gitops.SyncModeDeploy && mode != gitops.SyncModeBackup:
		return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "mode", Err: fmt.Errorf("unsupported sync mode %q", req.Mode)})
	case mode == gitops.SyncModeDeploy && req.HasBackupOptions():
		return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "mode", Err: errors.New("backup options require mode \"backup\"")})
	case mode == gitops.SyncModeDeploy && strings.TrimSpace(req.ComposePath) == "":
		return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "composePath", Err: errors.New("compose path is required")})
	}

	repo, err := s.repoService.GetRepositoryByID(ctx, req.RepositoryID)
	if err != nil {
		slog.ErrorContext(ctx, "Repository not found for GitOps sync", "repositoryId", req.RepositoryID, "error", err)
		return nil, fmt.Errorf("repository not found: %w", err)
	}
	slog.InfoContext(ctx, "Found repository for GitOps sync", "repositoryId", req.RepositoryID, "repositoryName", repo.Name)

	if limitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); limitsErr != nil {
		return nil, limitsErr
	}
	lifecycleCfg := lifecycleConfigInput{
		targetType:    &req.TargetType,
		scriptPath:    req.PreDeployScriptPath,
		runnerImage:   req.PreDeployRunnerImage,
		env:           req.PreDeployEnv,
		extraMounts:   req.PreDeployExtraMounts,
		timeoutSec:    req.PreDeployTimeoutSec,
		networkMode:   req.PreDeployNetworkMode,
		syncDirectory: req.SyncDirectory,
	}
	if lifecycleErr := s.validateLifecycleConfig(ctx, nil, lifecycleCfg); lifecycleErr != nil {
		return nil, lifecycleErr
	}

	defaultMaxFiles, defaultMaxTotalSize, defaultMaxBinarySize := s.getEnvironmentSyncLimits(ctx)
	syncRecord := projectpkg.GitOpsSync{
		Name:                 req.Name,
		EnvironmentID:        environmentID,
		RepositoryID:         req.RepositoryID,
		Branch:               req.Branch,
		ComposePath:          req.ComposePath,
		TargetType:           req.TargetType,
		ProjectName:          cmp.Or(req.ProjectName, req.Name),
		Mode:                 mode,
		AutoSync:             kit.FromPtr(req.AutoSync),
		SyncInterval:         *cmp.Or(req.SyncInterval, new(60)),
		SyncDirectory:        kit.FromPtr(req.SyncDirectory),
		PullImageAfterSync:   kit.FromPtr(req.PullImageAfterSync),
		RedeployAfterSync:    kit.FromPtr(req.RedeployAfterSync),
		MaxSyncFiles:         *cmp.Or(req.MaxSyncFiles, &defaultMaxFiles),
		MaxSyncTotalSize:     *cmp.Or(req.MaxSyncTotalSize, &defaultMaxTotalSize),
		MaxSyncBinarySize:    *cmp.Or(req.MaxSyncBinarySize, &defaultMaxBinarySize),
		BackupOnSave:         true,
		PreDeployScriptPath:  nullableTrimmedString(req.PreDeployScriptPath),
		PreDeployRunnerImage: nullableTrimmedString(req.PreDeployRunnerImage),
		PreDeployEnv:         nullableTrimmedString(req.PreDeployEnv),
		PreDeployExtraMounts: nullableTrimmedString(req.PreDeployExtraMounts),
		PreDeployTimeoutSec:  *cmp.Or(req.PreDeployTimeoutSec, new(lifecycle.DefaultTimeoutSec)),
		PreDeployNetworkMode: cmp.Or(strings.TrimSpace(kit.FromPtr(req.PreDeployNetworkMode)), "none"),
	}

	adoptedProject, err := s.insertSyncRecord(ctx, &syncRecord, req, mode)
	if err != nil {
		return nil, err
	}
	slog.InfoContext(ctx, "GitOps sync created successfully", "syncId", syncRecord.ID, "name", syncRecord.Name)
	if adoptedProject != nil {
		adoptedProject.GitOpsManagedBy = &syncRecord.ID
		if linkErr := s.projectService.EnsureGitOpsProjectLinked(ctx, &syncRecord, adoptedProject); linkErr != nil {
			return nil, fmt.Errorf("failed to link existing project: %w", linkErr)
		}
	}

	_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
		Type:          event.EventTypeGitSyncCreate,
		Severity:      event.EventSeveritySuccess,
		Title:         "Git sync created",
		Description:   fmt.Sprintf("Created git sync configuration '%s'", syncRecord.Name),
		ResourceType:  new("git_sync"),
		ResourceID:    new(syncRecord.ID),
		ResourceName:  new(syncRecord.Name),
		UserID:        new(actor.ID),
		Username:      new(actor.Username),
		EnvironmentID: new(syncRecord.EnvironmentID),
	})

	// The sync exists even when its first run fails, so that failure can be retried rather than failing creation.
	if _, syncErr := s.PerformSync(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor); syncErr != nil {
		slog.ErrorContext(ctx, "Failed to perform initial sync after creation", "syncId", syncRecord.ID, "error", syncErr)
	}
	// The initial sync above already ran, so the job needs no kick.
	if syncRecord.AutoSync {
		s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
	}
	return s.GetSyncByID(ctx, "", syncRecord.ID)
}

// insertSyncRecord inserts the sync in one transaction with any linked project locked, so a project never
// ends up with two Git relationships. It returns the project a deploy sync adopts.
func (s *GitOpsSyncService) insertSyncRecord(ctx context.Context, syncRecord *projectpkg.GitOpsSync, req gitops.CreateSyncRequest, mode string) (*projectpkg.Project, error) {
	var adoptedProject *projectpkg.Project
	err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		// An adopted project must exist, and must not be deployed from or backed up to Git already.
		if mode == gitops.SyncModeDeploy && strings.TrimSpace(req.ProjectID) != "" {
			if strings.TrimSpace(req.TargetType) == "swarm_stack" {
				return common.Classify(common.ErrValidation, &base.FieldError{Field: "projectId", Err: errors.New("an existing project cannot be linked to a swarm stack sync")})
			}
			project, err := projectpkg.LockProjectForSync(tx, req.ProjectID)
			if err != nil {
				return err
			}
			if strings.TrimSpace(kit.FromPtr(project.GitOpsManagedBy)) != "" {
				return common.Classify(common.ErrConflict, errors.New("project is already deployed from Git"))
			}
			var backups int64
			if countErr := tx.Model(&projectpkg.GitOpsSync{}).Where("mode = ? AND project_id = ?", gitops.SyncModeBackup, project.ID).Count(&backups).Error; countErr != nil {
				return fmt.Errorf("failed to check existing backups: %w", countErr)
			}
			if backups > 0 {
				return common.Classify(common.ErrConflict, errors.New("project is backed up to Git; disconnect that backup before deploying it from Git"))
			}
			if rootErr := s.projectService.EnsureProjectPathUnderRoot(ctx, project, false); rootErr != nil {
				return rootErr
			}
			adoptedProject = project
			syncRecord.ProjectID = &project.ID
			syncRecord.ProjectName = project.Name
		}
		if mode == gitops.SyncModeBackup {
			if err := s.backup.ApplyCreate(ctx, tx, req, syncRecord); err != nil {
				return err
			}
		}

		// Select("*") persists explicit zero values, such as unlimited sync limits, instead of column defaults.
		if err := tx.Select("*").Omit("Environment", "Repository", "Project").Create(syncRecord).Error; err != nil { //nolint:unqueryvet // intentional Select("*"); see comment above
			message := strings.ToLower(err.Error())
			if errors.Is(err, gorm.ErrDuplicatedKey) || strings.Contains(message, "unique constraint") || strings.Contains(message, "duplicate key") {
				return common.Classify(common.ErrConflict, errors.New("project already has a Git backup; disconnect it first"))
			}
			slog.ErrorContext(ctx, "Failed to create GitOps sync in database", "name", req.Name, "repositoryId", req.RepositoryID, "environmentId", syncRecord.EnvironmentID, "error", err)
			return fmt.Errorf("failed to create sync: %w", err)
		}
		if adoptedProject == nil {
			return nil
		}
		linked := tx.Model(&projectpkg.Project{}).
			Where("id = ? AND (gitops_managed_by IS NULL OR gitops_managed_by = '')", adoptedProject.ID).
			Update("gitops_managed_by", syncRecord.ID)
		if linked.Error != nil {
			return fmt.Errorf("failed to link existing project: %w", linked.Error)
		}
		if linked.RowsAffected != 1 {
			return common.Classify(common.ErrConflict, errors.New("project is already deployed from Git"))
		}
		return nil
	})
	return adoptedProject, err
}

func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id string, req gitops.UpdateSyncRequest, actor user.Actor) (*projectpkg.GitOpsSync, error) {
	syncRecord, err := s.GetSyncByID(ctx, environmentID, id)
	if err != nil {
		return nil, err
	}

	// Captured before the update so the job can follow the new state.
	oldAutoSync := syncRecord.AutoSync
	newAutoSync := *cmp.Or(req.AutoSync, &syncRecord.AutoSync)
	newInterval := *cmp.Or(req.SyncInterval, &syncRecord.SyncInterval)

	updates := make(map[string]any)

	if req.Name != nil {
		updates["name"] = *req.Name
	}
	if req.RepositoryID != nil {
		if _, repoErr := s.repoService.GetRepositoryByID(ctx, *req.RepositoryID); repoErr != nil {
			return nil, fmt.Errorf("repository not found: %w", repoErr)
		}
		updates["repository_id"] = *req.RepositoryID
	}
	if req.Branch != nil {
		updates["branch"] = *req.Branch
	}
	if req.ComposePath != nil {
		updates["compose_path"] = *req.ComposePath
	}
	if req.TargetType != nil {
		updates["target_type"] = *req.TargetType
	}
	if req.ProjectName != nil {
		updates["project_name"] = *req.ProjectName
	}
	if req.AutoSync != nil {
		updates["auto_sync"] = *req.AutoSync
	}
	if req.SyncInterval != nil {
		updates["sync_interval"] = *req.SyncInterval
	}
	if req.SyncDirectory != nil {
		updates["sync_directory"] = *req.SyncDirectory
	}
	if req.PullImageAfterSync != nil {
		updates["pull_image_after_sync"] = *req.PullImageAfterSync
	}
	if req.RedeployAfterSync != nil {
		updates["redeploy_after_sync"] = *req.RedeployAfterSync
	}
	if limitsErr := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); limitsErr != nil {
		return nil, limitsErr
	}
	if req.MaxSyncFiles != nil {
		updates["max_sync_files"] = *req.MaxSyncFiles
	}
	if req.MaxSyncTotalSize != nil {
		updates["max_sync_total_size"] = *req.MaxSyncTotalSize
	}
	if req.MaxSyncBinarySize != nil {
		updates["max_sync_binary_size"] = *req.MaxSyncBinarySize
	}

	if modeErr := s.backup.ApplyModeUpdates(ctx, syncRecord, req, updates); modeErr != nil {
		return nil, modeErr
	}

	lifecycleCfg := lifecycleConfigInput{
		targetType:    req.TargetType,
		scriptPath:    req.PreDeployScriptPath,
		runnerImage:   req.PreDeployRunnerImage,
		env:           req.PreDeployEnv,
		extraMounts:   req.PreDeployExtraMounts,
		timeoutSec:    req.PreDeployTimeoutSec,
		networkMode:   req.PreDeployNetworkMode,
		syncDirectory: req.SyncDirectory,
	}
	if lifecycleErr := s.validateLifecycleConfig(ctx, syncRecord, lifecycleCfg); lifecycleErr != nil {
		return nil, lifecycleErr
	}
	// A nil field stays unchanged; an empty one clears the column, or resets the network mode to "none".
	for column, value := range map[string]*string{
		"pre_deploy_script_path":  lifecycleCfg.scriptPath,
		"pre_deploy_runner_image": lifecycleCfg.runnerImage,
		"pre_deploy_env":          lifecycleCfg.env,
		"pre_deploy_extra_mounts": lifecycleCfg.extraMounts,
	} {
		if value != nil {
			updates[column] = nullableTrimmedString(value)
		}
	}
	if lifecycleCfg.timeoutSec != nil {
		updates["pre_deploy_timeout_sec"] = *lifecycleCfg.timeoutSec
	}
	if lifecycleCfg.networkMode != nil {
		updates["pre_deploy_network_mode"] = cmp.Or(strings.TrimSpace(*lifecycleCfg.networkMode), "none")
	}

	if len(updates) > 0 {
		// Loaded associations must not overwrite explicitly updated foreign keys.
		if updateErr := s.db.WithContext(ctx).Model(syncRecord).Omit(clause.Associations).Updates(updates).Error; updateErr != nil {
			return nil, fmt.Errorf("failed to update sync: %w", updateErr)
		}
		_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
			Type:          event.EventTypeGitSyncUpdate,
			Severity:      event.EventSeveritySuccess,
			Title:         "Git sync updated",
			Description:   fmt.Sprintf("Updated git sync configuration '%s'", syncRecord.Name),
			ResourceType:  new("git_sync"),
			ResourceID:    new(syncRecord.ID),
			ResourceName:  new(syncRecord.Name),
			UserID:        new(actor.ID),
			Username:      new(actor.Username),
			EnvironmentID: new(syncRecord.EnvironmentID),
		})
	}

	if !newAutoSync {
		s.jobs.Unregister(ctx, syncRecord.ID)
		return s.GetSyncByID(ctx, environmentID, id)
	}
	s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, newInterval)
	if !oldAutoSync {
		// A freshly enabled sync runs now instead of waiting a full interval.
		s.kickSync(ctx, syncRecord.ID, "startup")
	}

	return s.GetSyncByID(ctx, environmentID, id)
}

func (s *GitOpsSyncService) DeleteSync(ctx context.Context, environmentID, id string, actor user.Actor) error {
	// Even a row that can no longer be loaded must stop firing; an in-flight run re-reads it and skips.
	s.jobs.Unregister(ctx, id)
	s.backups.cancel(id)

	// The row is loaded only for the audit event, so a corrupt or environment-mismatched row is still deletable.
	syncRecord, loadErr := s.getSyncByID(ctx, environmentID, id, false)

	if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		// Keyed on the sync ID, so orphaned managed flags clear even when the row could not be loaded.
		if err := tx.Model(&projectpkg.Project{}).
			Where("gitops_managed_by = ?", id).
			Update("gitops_managed_by", nil).Error; err != nil {
			return fmt.Errorf("failed to clear gitops_managed_by: %w", err)
		}
		// The handler enforced the delete permission, so the delete is not environment-scoped, and deleting nothing succeeds.
		if err := tx.Where("id = ?", id).Delete(&projectpkg.GitOpsSync{}).Error; err != nil {
			return fmt.Errorf("failed to delete sync: %w", err)
		}
		return nil
	}); err != nil {
		if loadErr == nil && syncRecord.AutoSync {
			s.registerSyncJob(ctx, syncRecord.ID, syncRecord.EnvironmentID, syncRecord.SyncInterval)
		}
		return err
	}

	if loadErr != nil {
		slog.WarnContext(ctx, "Deleted GitOps sync whose record could not be loaded", "syncId", id, "loadError", loadErr)
		return nil
	}

	_, _ = s.eventService.CreateEvent(ctx, event.CreateEventRequest{
		Type:          event.EventTypeGitSyncDelete,
		Severity:      event.EventSeverityInfo,
		Title:         "Git sync deleted",
		Description:   fmt.Sprintf("Deleted git sync configuration '%s'", syncRecord.Name),
		ResourceType:  new("git_sync"),
		ResourceID:    new(syncRecord.ID),
		ResourceName:  new(syncRecord.Name),
		UserID:        new(actor.ID),
		Username:      new(actor.Username),
		EnvironmentID: new(syncRecord.EnvironmentID),
	})

	return nil
}

func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id string, actor user.Actor) (*gitops.SyncResult, error) {
	return s.performSyncAdmitted(ctx, environmentID, id, actor, false)
}

// performSyncAdmitted runs one sync under its admission lease and dispatches by mode. backupAdopt makes a
// backup replace whatever the remote holds in its backup directory.
func (s *GitOpsSyncService) performSyncAdmitted(ctx context.Context, environmentID, id string, actor user.Actor, backupAdopt bool) (syncResult *gitops.SyncResult, syncErr error) {
	ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "gitops.sync", trace.WithAttributes(
		attribute.String("arcane.gitops.sync_id", id),
		attribute.String("arcane.environment.id", environmentID),
	))
	counted := false
	defer func() {
		success := syncErr == nil && syncResult != nil && syncResult.Success
		if counted {
			outcome := attribute.String("arcane.gitops.outcome", kit.Ternary(success, "success", "failure"))
			span.SetAttributes(outcome)
			syncRuns.Add(context.WithoutCancel(ctx), 1, metric.WithAttributes(outcome))
		}
		tracing.End(span, syncErr)
	}()

	// Overlapping runs of one sync (schedule, kick, manual, webhook) must not race the clone and redeploy.
	lease, admitted, err := s.jobs.TryAcquire(ctx, id)
	if err != nil {
		return nil, fmt.Errorf("failed to admit GitOps sync: %w", err)
	}
	if !admitted {
		slog.InfoContext(ctx, "GitOps sync already in progress; skipping", "syncId", id)
		span.SetAttributes(attribute.Bool("arcane.gitops.skipped", true))
		return &gitops.SyncResult{Success: false, Message: "sync already in progress", SyncedAt: time.Now()}, nil
	}
	defer lease.Release(ctx)
	counted = true

	syncCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
	defer cancel()

	syncRecord, err := s.GetSyncByID(syncCtx, environmentID, id)
	if err != nil {
		return nil, err
	}
	span.SetAttributes(
		attribute.String("arcane.git.branch", syncRecord.Branch),
		attribute.String("arcane.gitops.mode", syncRecord.Mode),
		attribute.String("arcane.gitops.target_type", syncRecord.TargetType),
	)
	if syncRecord.Repository != nil {
		span.SetAttributes(git.RepositoryAttribute(syncRecord.Repository.URL))
	}

	result := &gitops.SyncResult{
		Success:  false,
		SyncedAt: time.Now(),
	}

	if syncRecord.Mode == gitops.SyncModeBackup {
		return s.backup.Perform(syncCtx, syncRecord, actor, result, backupAdopt)
	}

	return s.sync.Run(ctx, defaultGitSyncTimeout, syncRecord, actor, result)
}

func (s *GitOpsSyncService) GetSyncStatus(ctx context.Context, environmentID, id string) (*gitops.SyncStatus, error) {
	syncRecord, err := s.GetSyncByID(ctx, environmentID, id)
	if err != nil {
		return nil, err
	}

	return &gitops.SyncStatus{
		ID:                  syncRecord.ID,
		AutoSync:            syncRecord.AutoSync,
		LastSyncAt:          syncRecord.LastSyncAt,
		LastSyncStatus:      syncRecord.LastSyncStatus,
		LastSyncError:       syncRecord.LastSyncError,
		LastSyncCommit:      syncRecord.LastSyncCommit,
		Mode:                syncRecord.Mode,
		BackupState:         syncRecord.BackupState(),
		BackupPending:       syncRecord.BackupPending,
		BackupFailureReason: syncRecord.BackupFailureReason,
		LastBackupAt:        syncRecord.LastBackupAt,
		NextSyncAt: func() *time.Time {
			if !syncRecord.AutoSync || syncRecord.LastSyncAt == nil {
				return nil
			}
			return new(syncRecord.LastSyncAt.Add(time.Duration(syncRecord.SyncInterval) * time.Minute))
		}(),
	}, nil
}

// CleanupLeakedScratchDirsOnStartup removes GitOps scratch directories an interrupted sync left in the projects
// directory, which discovery would import as phantom projects. It runs before any sync job is registered.
func (s *GitOpsSyncService) CleanupLeakedScratchDirsOnStartup(ctx context.Context) error {
	projectsDir, err := s.projectService.GetProjectsDirectory(ctx)
	if err != nil {
		return fmt.Errorf("failed to resolve projects directory for gitops scratch cleanup: %w", err)
	}

	entries, err := acfs.List(ctx, projectsDir, "/")
	if err != nil {
		if errors.Is(err, fs.ErrNotExist) {
			return nil
		}
		return fmt.Errorf("failed to list projects directory %s for gitops scratch cleanup: %w", projectsDir, err)
	}

	removed := 0
	for _, entry := range entries {
		if !entry.IsDirectory || !projects.IsGitOpsScratchDirName(entry.Name) {
			continue
		}
		scratchPath := filepath.Join(projectsDir, entry.Name)
		if rmErr := acfs.RemoveAll(ctx, projectsDir, entry.Path); rmErr != nil {
			slog.WarnContext(ctx, "Failed to remove leaked GitOps scratch directory on startup", "path", scratchPath, "error", rmErr)
			continue
		}
		removed++
		slog.InfoContext(ctx, "Removed leaked GitOps scratch directory on startup", "path", scratchPath)
	}
	if removed > 0 {
		slog.InfoContext(ctx, "Cleaned up leaked GitOps scratch directories on startup", "count", removed)
	}
	return nil
}

// CleanupLeakedCloneDirsOnStartup removes leaked git clone scratch dirs
// ("gitops-*") under the git work dir; safe because no sync holds a clone yet.
func (s *GitOpsSyncService) CleanupLeakedCloneDirsOnStartup(ctx context.Context) error {
	if s.repoService == nil || s.repoService.Client == nil {
		return nil
	}

	removed, err := s.repoService.PurgeScratchDirs(ctx, 0)
	if err != nil {
		return fmt.Errorf("failed to purge leaked git clone scratch directories: %w", err)
	}
	if removed > 0 {
		slog.InfoContext(ctx, "Cleaned up leaked git clone scratch directories on startup", "count", removed)
	}
	return nil
}

func (s *GitOpsSyncService) CleanupOrphanedSyncsOnStartup(ctx context.Context) error {
	var syncIDs []string
	if err := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).
		Where("environment_id NOT IN (SELECT id FROM environments)").
		Pluck("id", &syncIDs).Error; err != nil {
		return fmt.Errorf("failed to list orphaned gitops syncs: %w", err)
	}
	if len(syncIDs) == 0 {
		return nil
	}

	if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
		if err := tx.Model(&projectpkg.Project{}).
			Where("gitops_managed_by IN ?", syncIDs).
			Update("gitops_managed_by", nil).Error; err != nil {
			return fmt.Errorf("failed to clear orphaned gitops project references: %w", err)
		}
		if err := tx.Where("id IN ?", syncIDs).Delete(&projectpkg.GitOpsSync{}).Error; err != nil {
			return fmt.Errorf("failed to delete orphaned gitops syncs: %w", err)
		}
		return nil
	}); err != nil {
		return err
	}

	slog.InfoContext(ctx, "Cleaned up orphaned GitOps syncs on startup", "syncIds", syncIDs)
	return nil
}

func (s *GitOpsSyncService) ReconcileDirectorySyncProjectsOnStartup(ctx context.Context) error {
	var syncs []projectpkg.GitOpsSync
	if err := s.db.WithContext(ctx).
		Where("sync_directory = ?", true).
		Find(&syncs).Error; err != nil {
		return fmt.Errorf("failed to list directory syncs for startup reconciliation: %w", err)
	}

	for i := range syncs {
		originalProjectID := ""
		if syncs[i].ProjectID != nil {
			originalProjectID = *syncs[i].ProjectID
		}

		project, err := s.sync.DirectoryProject(ctx, &syncs[i])
		if err != nil {
			slog.WarnContext(ctx, "Failed to reconcile directory GitOps sync on startup", "syncId", syncs[i].ID, "error", err)
			continue
		}
		if project == nil {
			continue
		}

		if originalProjectID != project.ID {
			slog.InfoContext(ctx, "Reconciled directory GitOps sync on startup", "syncId", syncs[i].ID, "projectId", project.ID)
		}
	}

	return nil
}

func (s *GitOpsSyncService) BrowseFiles(ctx context.Context, environmentID, id, localPath string) (*gitops.BrowseResponse, error) {
	browseCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
	defer cancel()

	syncRecord, err := s.GetSyncByID(browseCtx, environmentID, id)
	if err != nil {
		return nil, err
	}

	repository := syncRecord.Repository
	if repository == nil {
		return nil, errors.New("repository not found")
	}

	authConfig, err := s.repoService.GetAuthConfig(browseCtx, repository)
	if err != nil {
		return nil, err
	}

	// Clone the repository
	repoPath, err := s.repoService.Clone(browseCtx, repository.URL, syncRecord.Branch, authConfig, 1)
	if err != nil {
		return nil, fmt.Errorf("failed to clone repository: %w", err)
	}
	defer s.repoService.Discard(browseCtx, repoPath)

	// Browse the tree
	files, err := s.repoService.BrowseTree(browseCtx, repoPath, localPath)
	if err != nil {
		return nil, err
	}

	return &gitops.BrowseResponse{
		Path:  localPath,
		Files: files,
	}, nil
}

func (s *GitOpsSyncService) ImportSyncs(ctx context.Context, environmentID string, req []gitops.ImportGitOpsSyncRequest, actor user.Actor) (*gitops.ImportGitOpsSyncResponse, error) {
	response := &gitops.ImportGitOpsSyncResponse{
		SuccessCount: 0,
		FailedCount:  0,
		Errors:       []string{},
	}

	for _, importItem := range req {
		// Find repository by name
		repo, err := s.repoService.GetRepositoryByName(ctx, importItem.GitRepo)
		if err != nil {
			response.FailedCount++
			response.Errors = append(response.Errors, fmt.Sprintf("Sync '%s': Repository '%s' not found (%v)", importItem.SyncName, importItem.GitRepo, err))
			continue
		}

		createReq := gitops.CreateSyncRequest{
			Name:                   importItem.SyncName,
			RepositoryID:           repo.ID,
			Branch:                 importItem.Branch,
			ComposePath:            importItem.DockerComposePath,
			ProjectName:            importItem.ProjectName,
			AutoSync:               new(importItem.AutoSync),
			SyncInterval:           new(importItem.SyncInterval),
			SyncDirectory:          importItem.SyncDirectory,
			PullImageAfterSync:     importItem.PullImageAfterSync,
			RedeployAfterSync:      importItem.RedeployAfterSync,
			MaxSyncFiles:           importItem.MaxSyncFiles,
			MaxSyncTotalSize:       importItem.MaxSyncTotalSize,
			MaxSyncBinarySize:      importItem.MaxSyncBinarySize,
			PreDeployConfigRequest: importItem.PreDeployConfigRequest,
		}

		_, err = s.CreateSync(ctx, environmentID, createReq, actor)
		if err != nil {
			response.FailedCount++
			response.Errors = append(response.Errors, fmt.Sprintf("Sync '%s': %v", importItem.SyncName, err))
		} else {
			response.SuccessCount++
		}
	}

	return response, nil
}

func (r *backupRuntime) branchLock(repositoryID, branch string) *sync.Mutex {
	r.mu.Lock()
	defer r.mu.Unlock()
	key := repositoryID + "\x00" + branch
	lock, ok := r.branches[key]
	if !ok {
		lock = &sync.Mutex{}
		r.branches[key] = lock
	}
	return lock
}

func (r *backupRuntime) schedule(syncID string, run func()) {
	r.mu.Lock()
	defer r.mu.Unlock()
	if timer, ok := r.timers[syncID]; ok {
		timer.Stop()
	}
	r.timers[syncID] = time.AfterFunc(r.debounce, func() {
		defer utils.RecoverToError(nil, "git backup debounce", "syncId", syncID)
		r.mu.Lock()
		delete(r.timers, syncID)
		r.mu.Unlock()
		run()
	})
}

func (r *backupRuntime) cancel(syncID string) {
	r.mu.Lock()
	defer r.mu.Unlock()
	if timer, ok := r.timers[syncID]; ok {
		timer.Stop()
		delete(r.timers, syncID)
	}
}

// ResolveBackupConflict applies the chosen strategy to a backup that needs attention.
func (s *GitOpsSyncService) ResolveBackupConflict(ctx context.Context, environmentID, id string, req gitops.ResolveBackupConflictRequest, actor user.Actor) (*gitops.SyncResult, error) {
	if req.Strategy != gitops.BackupConflictUseArcane {
		return nil, common.Classify(common.ErrValidation, &base.FieldError{Field: "strategy", Err: fmt.Errorf("unsupported strategy %q", req.Strategy)})
	}
	syncRecord, err := s.getBackupSync(ctx, environmentID, id)
	if err != nil {
		return nil, err
	}
	return s.performSyncAdmitted(ctx, syncRecord.EnvironmentID, syncRecord.ID, actor, true)
}

// SubscribeProjectFileChanges marks backups pending on save and runs them after a debounce when auto backup is on.
func (s *GitOpsSyncService) SubscribeProjectFileChanges(ctx context.Context) {
	if s.projectService == nil {
		return
	}
	runCtx := s.jobs.Context(ctx)
	s.projectService.FilesChanged.Subscribe(func(projectID string) {
		var syncs []projectpkg.GitOpsSync
		if err := s.db.WithContext(runCtx).
			Where("mode = ? AND project_id = ?", gitops.SyncModeBackup, projectID).
			Find(&syncs).Error; err != nil {
			slog.ErrorContext(runCtx, "Failed to load git backups for changed project", "projectId", projectID, "error", err)
			return
		}
		now := time.Now()
		for _, syncRecord := range syncs {
			if err := s.db.WithContext(runCtx).Model(&projectpkg.GitOpsSync{}).Where("id = ?", syncRecord.ID).
				Updates(map[string]any{"backup_pending": true, "backup_pending_since": now}).Error; err != nil {
				slog.ErrorContext(runCtx, "Failed to mark git backup pending", "syncId", syncRecord.ID, "error", err)
				continue
			}
			if !syncRecord.AutoSync || !syncRecord.BackupOnSave {
				continue
			}
			syncID, environmentID := syncRecord.ID, syncRecord.EnvironmentID
			s.backups.schedule(syncID, func() {
				if s.jobs.Enabled() {
					s.kickSync(runCtx, syncID, "save")
					return
				}
				go func() {
					defer utils.RecoverToError(nil, "git backup after save", "syncId", syncID, "projectId", projectID)
					if _, err := s.PerformSync(runCtx, environmentID, syncID, user.SystemUser); err != nil {
						slog.ErrorContext(runCtx, "git backup after save failed", "syncId", syncID, "error", err)
					}
				}()
			})
		}
	})
}

// PreviewBackup reports what the next backup run would commit or why it needs attention.
func (s *GitOpsSyncService) PreviewBackup(ctx context.Context, environmentID, id string) (*gitops.BackupPreview, error) {
	previewCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
	defer cancel()

	syncRecord, err := s.getBackupSync(previewCtx, environmentID, id)
	if err != nil {
		return nil, err
	}
	return s.backup.Preview(previewCtx, syncRecord)
}

// GetBackupHistory lists revisions that touched the backup directory.
func (s *GitOpsSyncService) GetBackupHistory(ctx context.Context, environmentID, id string, limit int) (*gitops.BackupHistoryResponse, error) {
	historyCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
	defer cancel()

	syncRecord, err := s.getBackupSync(historyCtx, environmentID, id)
	if err != nil {
		return nil, err
	}
	return s.backup.History(historyCtx, syncRecord, limit)
}

// GetBackupRevision returns one revision with per-file diffs inside the backup directory.
func (s *GitOpsSyncService) GetBackupRevision(ctx context.Context, environmentID, id, commit string) (*gitops.BackupRevision, error) {
	revisionCtx, cancel := context.WithTimeout(ctx, defaultGitSyncTimeout)
	defer cancel()

	syncRecord, err := s.getBackupSync(revisionCtx, environmentID, id)
	if err != nil {
		return nil, err
	}
	return s.backup.Revision(revisionCtx, syncRecord, commit)
}

// ReconcileInterruptedBackupsOnStartup turns backups left running by a restart into pending failures.
func (s *GitOpsSyncService) ReconcileInterruptedBackupsOnStartup(ctx context.Context, protectedIDs ...string) error {
	query := s.db.WithContext(ctx).Model(&projectpkg.GitOpsSync{}).Where("mode = ? AND last_sync_status = ?", gitops.SyncModeBackup, backup.BackupStatusRunning)
	if len(protectedIDs) > 0 {
		query = query.Where("id NOT IN ?", protectedIDs)
	}
	err := query.Updates(map[string]any{
		"last_sync_status":      "failed",
		"last_sync_error":       "backup was interrupted by a restart",
		"backup_failure_reason": gitops.BackupFailureRepository,
		"backup_pending":        true,
		"backup_pending_since":  time.Now(),
	}).Error
	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
		return fmt.Errorf("failed to reconcile interrupted git backups: %w", err)
	}
	return nil
}

func (s *GitOpsSyncService) getBackupSync(ctx context.Context, environmentID, id string) (*projectpkg.GitOpsSync, error) {
	syncRecord, err := s.GetSyncByID(ctx, environmentID, id)
	if err != nil {
		return nil, err
	}
	if syncRecord.Mode != gitops.SyncModeBackup {
		return nil, common.Classify(common.ErrBadRequest, errors.New("sync does not back up to Git"))
	}
	if syncRecord.Repository == nil {
		return nil, common.Classify(common.ErrNotFound, errors.New("repository not found"))
	}
	return syncRecord, nil
}