Skip to content

Modern Docker Management, Designed for Everyone

feat: Git Sync Folder Implementation (#1629)

Alexander67b693766dbc
diff --git a/backend/internal/bootstrap/services_bootstrap.go b/backend/internal/bootstrap/services_bootstrap.go
index 2100c4e08..0b9722ae6 100644
--- a/backend/internal/bootstrap/services_bootstrap.go
+++ b/backend/internal/bootstrap/services_bootstrap.go
@@ -90,7 +90,7 @@ func initializeServices(ctx context.Context, db *database.DB, cfg *config.Config
svcs.Version = services.NewVersionService(httpClient, cfg.UpdateCheckDisabled, config.Version, config.Revision, svcs.ContainerRegistry, svcs.Docker)
svcs.SystemUpgrade = services.NewSystemUpgradeService(svcs.Docker, svcs.Version, svcs.Event, svcs.Settings)
svcs.Updater = services.NewUpdaterService(db, svcs.Settings, svcs.Docker, svcs.Project, svcs.ImageUpdate, svcs.ContainerRegistry, svcs.Event, svcs.Image, svcs.Notification, svcs.SystemUpgrade)
- svcs.GitOpsSync = services.NewGitOpsSyncService(db, svcs.GitRepository, svcs.Project, svcs.Event)
+ svcs.GitOpsSync = services.NewGitOpsSyncService(db, svcs.GitRepository, svcs.Project, svcs.Event, svcs.Settings)
return svcs, dockerClient, nil
}
diff --git a/backend/internal/database/database_test.go b/backend/internal/database/database_test.go
index f0bbf6cbd..e139fddea 100644
--- a/backend/internal/database/database_test.go
+++ b/backend/internal/database/database_test.go
@@ -117,11 +117,16 @@ func TestMigrateDatabase_DirtyOlderVersionRequiresResolution(t *testing.T) {
driver := newSQLiteMigrationDriverInternal(t, dbDir, "arcane-test.db")
require.NoError(t, migrateDatabase(driver, "sqlite", MigrationOptions{}))
targetVersion := downgradeTargetVersionInternal(t)
+ highestVersion, err := getHighestEmbeddedMigrationVersionInternal("sqlite")
+ require.NoError(t, err)
+ sourceDriver, err := newEmbeddedMigrationSourceInternal("sqlite")
+ require.NoError(t, err)
+
+ require.NoError(t, migrateDatabaseFromSourceInternal(newSQLiteMigrationDriverInternal(t, dbDir, "arcane-test.db"), "sqlite", highestVersion, targetVersion, "iofs", "test embedded migrate source", sourceDriver))
+
targetVersionInt, err := safeUintToIntInternal(targetVersion)
require.NoError(t, err)
require.NoError(t, newSQLiteMigrationDriverInternal(t, dbDir, "arcane-test.db").SetVersion(targetVersionInt, true))
- highestVersion, err := getHighestEmbeddedMigrationVersionInternal("sqlite")
- require.NoError(t, err)
err = migrateDatabaseToVersionInternal(newSQLiteMigrationDriverInternal(t, dbDir, "arcane-test.db"), "sqlite", MigrationOptions{}, highestVersion)
require.Error(t, err)
diff --git a/backend/internal/models/gitops_sync.go b/backend/internal/models/gitops_sync.go
index 196bb9284..9530e2b3c 100644
--- a/backend/internal/models/gitops_sync.go
+++ b/backend/internal/models/gitops_sync.go
@@ -5,22 +5,27 @@ import (
)
type GitOpsSync struct {
- Name string `json:"name" sortable:"true" search:"sync,gitops,automation,deploy,deployment,continuous"`
- EnvironmentID string `json:"environmentId" sortable:"true"`
- Environment *Environment `json:"environment,omitempty" gorm:"foreignKey:EnvironmentID"`
- RepositoryID string `json:"repositoryId" sortable:"true"`
- Repository *GitRepository `json:"repository,omitempty" gorm:"foreignKey:RepositoryID"`
- Branch string `json:"branch" sortable:"true" search:"branch,main,master,develop,feature,release"`
- ComposePath string `json:"composePath" sortable:"true" search:"compose,docker-compose,path,file,yaml,yml"`
- ProjectName string `json:"projectName" sortable:"true" search:"project,name,stack,application,service"` // Name of project to create/update
- ProjectID *string `json:"projectId,omitempty" sortable:"true"` // Set after project is created
- Project *Project `json:"project,omitempty" gorm:"foreignKey:ProjectID"`
- AutoSync bool `json:"autoSync" sortable:"true" search:"auto,automatic,sync,continuous,scheduled"`
- SyncInterval int `json:"syncInterval" sortable:"true" search:"interval,frequency,schedule,cron,minutes"` // in minutes
- LastSyncAt *time.Time `json:"lastSyncAt,omitempty" sortable:"true"`
- LastSyncStatus *string `json:"lastSyncStatus,omitempty" search:"status,success,failed,pending,error"`
- LastSyncError *string `json:"lastSyncError,omitempty"`
- LastSyncCommit *string `json:"lastSyncCommit,omitempty" search:"commit,hash,sha,revision"`
+ Name string `json:"name" sortable:"true" search:"sync,gitops,automation,deploy,deployment,continuous"`
+ EnvironmentID string `json:"environmentId" sortable:"true"`
+ Environment *Environment `json:"environment,omitempty" gorm:"foreignKey:EnvironmentID"`
+ RepositoryID string `json:"repositoryId" sortable:"true"`
+ Repository *GitRepository `json:"repository,omitempty" gorm:"foreignKey:RepositoryID"`
+ Branch string `json:"branch" sortable:"true" search:"branch,main,master,develop,feature,release"`
+ ComposePath string `json:"composePath" sortable:"true" search:"compose,docker-compose,path,file,yaml,yml"`
+ ProjectName string `json:"projectName" sortable:"true" search:"project,name,stack,application,service"` // Name of project to create/update
+ ProjectID *string `json:"projectId,omitempty" sortable:"true"` // Set after project is created
+ Project *Project `json:"project,omitempty" gorm:"foreignKey:ProjectID"`
+ AutoSync bool `json:"autoSync" sortable:"true" search:"auto,automatic,sync,continuous,scheduled"`
+ SyncInterval int `json:"syncInterval" sortable:"true" search:"interval,frequency,schedule,cron,minutes"` // in minutes
+ SyncDirectory bool `json:"syncDirectory" gorm:"column:sync_directory;default:true"` // Sync entire directory containing compose file
+ SyncedFiles *string `json:"syncedFiles,omitempty" gorm:"column:synced_files"` // JSON array of synced file paths
+ MaxSyncFiles int `json:"maxSyncFiles" gorm:"column:max_sync_files;default:500"` // 0 = inherit environment limit; unlimited only if the effective environment limit is also 0
+ MaxSyncTotalSize int64 `json:"maxSyncTotalSize" gorm:"column:max_sync_total_size;default:52428800"` // bytes; 0 = inherit environment limit; unlimited only if the effective environment limit is also 0
+ MaxSyncBinarySize int64 `json:"maxSyncBinarySize" gorm:"column:max_sync_binary_size;default:10485760"` // bytes; 0 = inherit environment limit; unlimited only if the effective environment limit is also 0
+ LastSyncAt *time.Time `json:"lastSyncAt,omitempty" sortable:"true"`
+ LastSyncStatus *string `json:"lastSyncStatus,omitempty" search:"status,success,failed,pending,error"`
+ LastSyncError *string `json:"lastSyncError,omitempty"`
+ LastSyncCommit *string `json:"lastSyncCommit,omitempty" search:"commit,hash,sha,revision"`
BaseModel
}
diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.go
index 066ecce77..8afd685de 100644
--- a/backend/internal/models/settings.go
+++ b/backend/internal/models/settings.go
@@ -94,6 +94,9 @@ type Settings struct {
AutoHealMaxRestarts SettingVariable `key:"autoHealMaxRestarts" meta:"label=Auto Heal Max Restarts;type=number;keywords=auto,heal,max,restarts,limit,loop,protection;category=internal;description=Maximum auto-heal restarts per container within the restart window (default: 5)"`
AutoHealRestartWindow SettingVariable `key:"autoHealRestartWindow" meta:"label=Auto Heal Restart Window;type=number;keywords=auto,heal,restart,window,minutes,cooldown,protection;category=internal;description=Time window in minutes for counting auto-heal restarts (default: 30)"`
MaxImageUploadSize SettingVariable `key:"maxImageUploadSize" meta:"label=Max Image Upload Size;type=number;keywords=upload,size,limit,maximum,image,tar,file,megabytes,mb,storage;category=internal;description=Maximum size in MB for image archive uploads (default: 500)"`
+ GitSyncMaxFiles SettingVariable `key:"gitSyncMaxFiles,envOverride" meta:"label=Git Sync Max Files;type=number;keywords=git,sync,files,limit,repository,compose,gitops;category=general;description=Maximum number of repository files copied during a Git sync. Set 0 to disable the environment cap (default: 500)"`
+ GitSyncMaxTotalSizeMb SettingVariable `key:"gitSyncMaxTotalSizeMb,envOverride" meta:"label=Git Sync Max Total Size (MB);type=number;keywords=git,sync,size,limit,repository,compose,gitops,mb;category=general;description=Maximum combined size in MB for files copied during a Git sync. Set 0 to disable the environment cap (default: 50)"`
+ GitSyncMaxBinarySizeMb SettingVariable `key:"gitSyncMaxBinarySizeMb,envOverride" meta:"label=Git Sync Max Binary Size (MB);type=number;keywords=git,sync,binary,size,limit,repository,compose,gitops,mb;category=general;description=Maximum size in MB for a single binary file copied during a Git sync. Set 0 to disable the environment cap (default: 10)"`
DockerHost SettingVariable `key:"dockerHost,public,envOverride" meta:"label=Docker Host;type=text;keywords=docker,host,daemon,socket,unix,remote;category=internal;description=URI for Docker daemon"`
BuildProvider SettingVariable `key:"buildProvider,envOverride" meta:"label=Build Provider;type=select;keywords=build,buildkit,depot,provider,remote,local;category=build;description=Default build provider (local or depot)" catmeta:"id=build;title=Build;icon=code;url=/settings/builds;description=Configure BuildKit and Depot build settings"`
BuildsDirectory SettingVariable `key:"buildsDirectory,envOverride" meta:"label=Builds Directory;type=text;keywords=builds,directory,path,workspace,context;category=build;description=Root directory for manual build workspaces"`
diff --git a/backend/internal/services/gitops_sync_service.go b/backend/internal/services/gitops_sync_service.go
index d049916f4..8252b0e44 100644
--- a/backend/internal/services/gitops_sync_service.go
+++ b/backend/internal/services/gitops_sync_service.go
@@ -2,6 +2,7 @@ package services
import (
"context"
+ "encoding/json"
"errors"
"fmt"
"log/slog"
@@ -21,14 +22,23 @@ import (
)
type GitOpsSyncService struct {
- db *database.DB
- repoService *GitRepositoryService
- projectService *ProjectService
- eventService *EventService
+ db *database.DB
+ repoService *GitRepositoryService
+ projectService *ProjectService
+ eventService *EventService
+ settingsService *SettingsService
}
const defaultGitSyncTimeout = 5 * time.Minute
+const (
+ defaultMaxSyncFiles = 500
+ defaultMaxSyncTotalSizeMB = 50
+ defaultMaxSyncBinarySizeMB = 10
+ defaultMaxSyncTotalSize = defaultMaxSyncTotalSizeMB * 1024 * 1024
+ defaultMaxSyncBinarySize = defaultMaxSyncBinarySizeMB * 1024 * 1024
+)
+
type scheduledGitOpsSync struct {
ID string
EnvironmentID string
@@ -36,13 +46,90 @@ type scheduledGitOpsSync struct {
LastSyncAt *time.Time
}
-func NewGitOpsSyncService(db *database.DB, repoService *GitRepositoryService, projectService *ProjectService, eventService *EventService) *GitOpsSyncService {
+func validateSyncLimits(maxFiles *int, maxTotalSize, maxBinarySize *int64) error {
+ if maxFiles != nil && *maxFiles < 0 {
+ return fmt.Errorf("maxSyncFiles must be non-negative")
+ }
+ if maxTotalSize != nil && *maxTotalSize < 0 {
+ return fmt.Errorf("maxSyncTotalSize must be non-negative")
+ }
+ if maxBinarySize != nil && *maxBinarySize < 0 {
+ return fmt.Errorf("maxSyncBinarySize must be non-negative")
+ }
+ return nil
+}
+
+func normalizeSyncLimitSetting(value, defaultValue int) int {
+ if value < 0 {
+ return defaultValue
+ }
+ return value
+}
+
+func megabytesToBytes(value int) int64 {
+ return int64(value) * 1024 * 1024
+}
+
+func effectiveIntLimit(syncLimit, environmentLimit int) int {
+ switch {
+ case syncLimit == 0:
+ return environmentLimit
+ case environmentLimit == 0:
+ return syncLimit
+ case syncLimit < environmentLimit:
+ return syncLimit
+ default:
+ return environmentLimit
+ }
+}
+
+func effectiveInt64Limit(syncLimit, environmentLimit int64) int64 {
+ switch {
+ case syncLimit == 0:
+ return environmentLimit
+ case environmentLimit == 0:
+ return syncLimit
+ case syncLimit < environmentLimit:
+ return syncLimit
+ default:
+ return environmentLimit
+ }
+}
+
+func NewGitOpsSyncService(db *database.DB, repoService *GitRepositoryService, projectService *ProjectService, eventService *EventService, settingsService *SettingsService) *GitOpsSyncService {
return &GitOpsSyncService{
- db: db,
- repoService: repoService,
- projectService: projectService,
- eventService: eventService,
+ db: db,
+ repoService: repoService,
+ projectService: projectService,
+ eventService: eventService,
+ settingsService: settingsService,
+ }
+}
+
+func (s *GitOpsSyncService) getEnvironmentSyncLimits(ctx context.Context) (int, int64, int64) {
+ if s.settingsService == nil {
+ return defaultMaxSyncFiles, defaultMaxSyncTotalSize, defaultMaxSyncBinarySize
}
+
+ maxFiles := normalizeSyncLimitSetting(s.settingsService.GetIntSetting(ctx, "gitSyncMaxFiles", defaultMaxSyncFiles), defaultMaxSyncFiles)
+ maxTotalSizeMB := normalizeSyncLimitSetting(
+ s.settingsService.GetIntSetting(ctx, "gitSyncMaxTotalSizeMb", defaultMaxSyncTotalSizeMB),
+ defaultMaxSyncTotalSizeMB,
+ )
+ maxBinarySizeMB := normalizeSyncLimitSetting(
+ s.settingsService.GetIntSetting(ctx, "gitSyncMaxBinarySizeMb", defaultMaxSyncBinarySizeMB),
+ defaultMaxSyncBinarySizeMB,
+ )
+
+ return maxFiles, megabytesToBytes(maxTotalSizeMB), megabytesToBytes(maxBinarySizeMB)
+}
+
+func (s *GitOpsSyncService) getEffectiveSyncLimits(ctx context.Context, sync *models.GitOpsSync) (int, int64, int64) {
+ environmentMaxFiles, environmentMaxTotalSize, environmentMaxBinarySize := s.getEnvironmentSyncLimits(ctx)
+
+ return effectiveIntLimit(sync.MaxSyncFiles, environmentMaxFiles),
+ effectiveInt64Limit(sync.MaxSyncTotalSize, environmentMaxTotalSize),
+ effectiveInt64Limit(sync.MaxSyncBinarySize, environmentMaxBinarySize)
}
func (s *GitOpsSyncService) ListSyncIntervalsRaw(ctx context.Context) ([]startup.IntervalMigrationItem, error) {
@@ -174,16 +261,22 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string
projectName = req.Name
}
+ defaultMaxFiles, defaultMaxTotalSize, defaultMaxBinarySize := s.getEnvironmentSyncLimits(ctx)
+
sync := models.GitOpsSync{
- Name: req.Name,
- EnvironmentID: environmentID,
- RepositoryID: req.RepositoryID,
- Branch: req.Branch,
- ComposePath: req.ComposePath,
- ProjectName: projectName,
- ProjectID: nil, // Will be set during first sync
- AutoSync: false,
- SyncInterval: 60,
+ Name: req.Name,
+ EnvironmentID: environmentID,
+ RepositoryID: req.RepositoryID,
+ Branch: req.Branch,
+ ComposePath: req.ComposePath,
+ ProjectName: projectName,
+ ProjectID: nil, // Will be set during first sync
+ AutoSync: false,
+ SyncInterval: 60,
+ SyncDirectory: true, // Default to directory sync
+ MaxSyncFiles: defaultMaxFiles,
+ MaxSyncTotalSize: defaultMaxTotalSize,
+ MaxSyncBinarySize: defaultMaxBinarySize,
}
if req.AutoSync != nil {
@@ -192,6 +285,21 @@ func (s *GitOpsSyncService) CreateSync(ctx context.Context, environmentID string
if req.SyncInterval != nil {
sync.SyncInterval = *req.SyncInterval
}
+ if req.SyncDirectory != nil {
+ sync.SyncDirectory = *req.SyncDirectory
+ }
+ if err := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); err != nil {
+ return nil, err
+ }
+ if req.MaxSyncFiles != nil {
+ sync.MaxSyncFiles = *req.MaxSyncFiles
+ }
+ if req.MaxSyncTotalSize != nil {
+ sync.MaxSyncTotalSize = *req.MaxSyncTotalSize
+ }
+ if req.MaxSyncBinarySize != nil {
+ sync.MaxSyncBinarySize = *req.MaxSyncBinarySize
+ }
if err := s.db.WithContext(ctx).Create(&sync).Error; err != nil {
slog.ErrorContext(ctx, "Failed to create GitOps sync in database", "name", req.Name, "repositoryID", req.RepositoryID, "environmentID", environmentID, "error", err)
@@ -255,6 +363,21 @@ func (s *GitOpsSyncService) UpdateSync(ctx context.Context, environmentID, id st
if req.SyncInterval != nil {
updates["sync_interval"] = *req.SyncInterval
}
+ if req.SyncDirectory != nil {
+ updates["sync_directory"] = *req.SyncDirectory
+ }
+ if err := validateSyncLimits(req.MaxSyncFiles, req.MaxSyncTotalSize, req.MaxSyncBinarySize); err != nil {
+ return nil, err
+ }
+ if req.MaxSyncFiles != nil {
+ updates["max_sync_files"] = *req.MaxSyncFiles
+ }
+ if req.MaxSyncTotalSize != nil {
+ updates["max_sync_total_size"] = *req.MaxSyncTotalSize
+ }
+ if req.MaxSyncBinarySize != nil {
+ updates["max_sync_binary_size"] = *req.MaxSyncBinarySize
+ }
if len(updates) > 0 {
if err := s.db.WithContext(ctx).Model(sync).Updates(updates).Error; err != nil {
@@ -388,17 +511,67 @@ func (s *GitOpsSyncService) PerformSync(ctx context.Context, environmentID, id s
}
}
- // Get or create project
- project, err := s.getOrCreateProjectInternal(syncCtx, sync, id, composeContent, envContent, result, actor)
- if err != nil {
- return result, err
- }
+ var project *models.Project
+ var syncedFiles []string
+
+ if sync.SyncDirectory {
+ // Directory sync mode - sync entire directory containing compose file
+ slog.InfoContext(syncCtx, "Using directory sync mode", "syncId", id, "composePath", sync.ComposePath)
+
+ // Walk directory once and get all files
+ var dirComposeContent string
+ var syncFiles []projects.SyncFile
+ var err error
+ dirComposeContent, syncFiles, err = s.walkAndParseSyncDirectory(syncCtx, sync, repoPath)
+ if err != nil {
+ return result, s.failSync(syncCtx, id, result, sync, actor, "Failed to walk directory", err.Error())
+ }
+
+ // Get or create project with compose content
+ project, err = s.getOrCreateProjectInternal(syncCtx, sync, id, dirComposeContent, nil, result, actor)
+ if err != nil {
+ return result, err
+ }
+
+ // Write all directory files to the project
+ oldSyncedFiles := parseSyncedFiles(sync.SyncedFiles)
+ syncedFiles, err = s.writeSyncFilesToProject(syncCtx, sync, project, syncFiles, oldSyncedFiles)
+ if err != nil {
+ slog.ErrorContext(syncCtx, "Failed to write directory files to project", "error", err, "syncId", id)
+ // Don't fail the sync - the project was created/updated with compose content
+ // Fall back to just tracking the file paths
+ syncedFiles = make([]string, len(syncFiles))
+ for i, f := range syncFiles {
+ syncedFiles[i] = f.RelativePath
+ }
+ }
- // Update sync status
- s.updateSyncStatus(syncCtx, id, "success", "", commitHash)
+ // Update sync status with synced files
+ s.updateSyncStatusWithFiles(syncCtx, id, "success", "", commitHash, syncedFiles)
+ } else {
+ // Single file sync mode - existing behavior
+ slog.InfoContext(syncCtx, "Using single file sync mode", "syncId", id, "composePath", sync.ComposePath)
+
+ // Get or create project (uses composeContent and envContent already read above)
+ var err error
+ project, err = s.getOrCreateProjectInternal(syncCtx, sync, id, composeContent, envContent, result, actor)
+ if err != nil {
+ return result, err
+ }
+
+ // Track single compose file as synced
+ syncedFiles = []string{filepath.Base(sync.ComposePath)}
+
+ // Update sync status with synced files
+ s.updateSyncStatusWithFiles(syncCtx, id, "success", "", commitHash, syncedFiles)
+ }
result.Success = true
- result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)
+ if sync.SyncDirectory {
+ result.Message = fmt.Sprintf("Successfully synced directory with %d files to project %s", len(syncedFiles), project.Name)
+ } else {
+ result.Message = fmt.Sprintf("Successfully synced compose file from %s to project %s", sync.ComposePath, project.Name)
+ }
// Log success event
_, _ = s.eventService.CreateEvent(syncCtx, CreateEventRequest{
@@ -558,13 +731,17 @@ func (s *GitOpsSyncService) ImportSyncs(ctx context.Context, environmentID strin
}
createReq := gitops.CreateSyncRequest{
- Name: importItem.SyncName,
- RepositoryID: repo.ID,
- Branch: importItem.Branch,
- ComposePath: importItem.DockerComposePath,
- ProjectName: importItem.SyncName,
- AutoSync: new(importItem.AutoSync),
- SyncInterval: new(importItem.SyncInterval),
+ Name: importItem.SyncName,
+ RepositoryID: repo.ID,
+ Branch: importItem.Branch,
+ ComposePath: importItem.DockerComposePath,
+ ProjectName: importItem.SyncName,
+ AutoSync: new(importItem.AutoSync),
+ SyncInterval: new(importItem.SyncInterval),
+ SyncDirectory: importItem.SyncDirectory,
+ MaxSyncFiles: importItem.MaxSyncFiles,
+ MaxSyncTotalSize: importItem.MaxSyncTotalSize,
+ MaxSyncBinarySize: importItem.MaxSyncBinarySize,
}
_, err = s.CreateSync(ctx, environmentID, createReq, actor)
@@ -694,3 +871,133 @@ func envContentChangedInternal(oldEnv, newEnv string) bool {
return !maps.Equal(oldEnvMap, newEnvMap)
}
+
+// getProjectsDirectory returns the configured projects directory path
+func (s *GitOpsSyncService) getProjectsDirectory(ctx context.Context) (string, error) {
+ projectsDirSetting := s.settingsService.GetStringSetting(ctx, "projectsDirectory", "/app/data/projects")
+ return projects.GetProjectsDirectory(ctx, strings.TrimSpace(projectsDirSetting))
+}
+
+// parseSyncedFiles parses the JSON array of synced file paths from the database
+func parseSyncedFiles(syncedFilesJSON *string) []string {
+ if syncedFilesJSON == nil || *syncedFilesJSON == "" {
+ return nil
+ }
+ var files []string
+ if err := json.Unmarshal([]byte(*syncedFilesJSON), &files); err != nil {
+ return nil
+ }
+ return files
+}
+
+// marshalSyncedFiles converts a list of file paths to JSON for storage
+func marshalSyncedFiles(files []string) *string {
+ if len(files) == 0 {
+ return nil
+ }
+ data, err := json.Marshal(files)
+ if err != nil {
+ return nil
+ }
+ result := string(data)
+ return &result
+}
+
+// walkAndParseSyncDirectory walks the repository directory and returns all files with their contents.
+// Returns the compose file content, the list of SyncFile entries, and an error if any.
+func (s *GitOpsSyncService) walkAndParseSyncDirectory(ctx context.Context, sync *models.GitOpsSync, repoPath string) (string, []projects.SyncFile, error) {
+ slog.InfoContext(ctx, "Starting directory walk", "syncId", sync.ID, "composePath", sync.ComposePath)
+
+ // Walk the directory to get all files
+ maxFiles, maxTotalSize, maxBinarySize := s.getEffectiveSyncLimits(ctx, sync)
+
+ walkResult, err := s.repoService.gitClient.WalkDirectory(ctx, repoPath, sync.ComposePath, maxFiles, maxTotalSize, maxBinarySize)
+ if err != nil {
+ return "", nil, fmt.Errorf("failed to walk directory: %w", err)
+ }
+
+ slog.InfoContext(ctx, "Directory walk complete",
+ "syncId", sync.ID,
+ "totalFiles", walkResult.TotalFiles,
+ "totalSize", walkResult.TotalSize,
+ "skippedBinaries", walkResult.SkippedBinaries)
+
+ // WalkDirectory roots the walk at filepath.Dir(sync.ComposePath), so the
+ // compose file is always emitted at the top level as filepath.Base(sync.ComposePath).
+ composeFileName := filepath.Base(sync.ComposePath)
+ var composeContent string
+
+ // Convert walked files to SyncFile format
+ syncFiles := make([]projects.SyncFile, len(walkResult.Files))
+ for i, f := range walkResult.Files {
+ syncFiles[i] = projects.SyncFile{
+ RelativePath: f.RelativePath,
+ Content: f.Content,
+ }
+ if f.RelativePath == composeFileName {
+ composeContent = string(f.Content)
+ }
+ }
+
+ if composeContent == "" {
+ return "", nil, fmt.Errorf("compose file %s not found in walked directory", composeFileName)
+ }
+
+ return composeContent, syncFiles, nil
+}
+
+// writeSyncFilesToProject writes the given sync files to the project directory.
+// If oldSyncedFiles is provided, removed files will be cleaned up first.
+func (s *GitOpsSyncService) writeSyncFilesToProject(ctx context.Context, sync *models.GitOpsSync, project *models.Project, syncFiles []projects.SyncFile, oldSyncedFiles []string) ([]string, error) {
+ projectsDir, err := s.getProjectsDirectory(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get projects directory: %w", err)
+ }
+
+ // Build list of new file paths
+ newFiles := make([]string, len(syncFiles))
+ for i, f := range syncFiles {
+ newFiles[i] = f.RelativePath
+ }
+
+ // Clean up removed files if we have old sync data
+ if len(oldSyncedFiles) > 0 {
+ if err := projects.CleanupRemovedFiles(projectsDir, project.Path, oldSyncedFiles, newFiles); err != nil {
+ slog.WarnContext(ctx, "Failed to cleanup removed files", "error", err, "syncId", sync.ID)
+ // Continue despite cleanup error - it's best effort
+ }
+ }
+
+ // Write all files to project directory
+ writtenPaths, err := projects.WriteSyncedDirectory(projectsDir, project.Path, syncFiles)
+ if err != nil {
+ return nil, fmt.Errorf("failed to write synced directory: %w", err)
+ }
+
+ slog.InfoContext(ctx, "Directory sync written", "syncId", sync.ID, "projectId", project.ID, "filesWritten", len(writtenPaths))
+ return writtenPaths, nil
+}
+
+// updateSyncStatusWithFiles updates sync status including the list of synced files
+func (s *GitOpsSyncService) updateSyncStatusWithFiles(ctx context.Context, id, status, errorMsg, commitHash string, syncedFiles []string) {
+ now := time.Now()
+ updates := map[string]interface{}{
+ "last_sync_at": now,
+ "last_sync_status": status,
+ "synced_files": marshalSyncedFiles(syncedFiles),
+ }
+
+ if errorMsg != "" {
+ updates["last_sync_error"] = errorMsg
+ } else {
+ updates["last_sync_error"] = nil
+ }
+
+ if commitHash != "" {
+ updates["last_sync_commit"] = commitHash
+ }
+
+ if err := s.db.WithContext(ctx).Model(&models.GitOpsSync{}).Where("id = ?", id).Updates(updates).Error; err != nil {
+ slog.ErrorContext(ctx, "Failed to update sync status with files", "error", err, "syncId", id)
+ }
+}
diff --git a/backend/internal/services/gitops_sync_service_test.go b/backend/internal/services/gitops_sync_service_test.go
index 5671b0a15..a1939e03a 100644
--- a/backend/internal/services/gitops_sync_service_test.go
+++ b/backend/internal/services/gitops_sync_service_test.go
@@ -1,9 +1,12 @@
package services
import (
+ "context"
"testing"
+ "github.com/getarcaneapp/arcane/backend/internal/models"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestEnvContentChangedInternal(t *testing.T) {
@@ -21,3 +24,77 @@ func TestEnvContentChangedInternal(t *testing.T) {
assert.True(t, envContentChangedInternal(oldEnv, newEnv))
})
}
+
+func TestGitOpsSyncService_GetEnvironmentSyncLimits(t *testing.T) {
+ ctx := context.Background()
+ db := setupSettingsTestDB(t)
+ settingsSvc, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 123))
+ require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxTotalSizeMb", 64))
+ require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxBinarySizeMb", 12))
+
+ svc := &GitOpsSyncService{settingsService: settingsSvc}
+
+ maxFiles, maxTotalSize, maxBinarySize := svc.getEnvironmentSyncLimits(ctx)
+
+ require.Equal(t, 123, maxFiles)
+ require.Equal(t, int64(64*1024*1024), maxTotalSize)
+ require.Equal(t, int64(12*1024*1024), maxBinarySize)
+}
+
+func TestGitOpsSyncService_GetEffectiveSyncLimits(t *testing.T) {
+ ctx := context.Background()
+ db := setupSettingsTestDB(t)
+ settingsSvc, err := NewSettingsService(ctx, db)
+ require.NoError(t, err)
+
+ require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxFiles", 200))
+ require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxTotalSizeMb", 30))
+ require.NoError(t, settingsSvc.SetIntSetting(ctx, "gitSyncMaxBinarySizeMb", 5))
+
+ svc := &GitOpsSyncService{settingsService: settingsSvc}
+
+ t.Run("uses environment caps when sync values are looser", func(t *testing.T) {
+ sync := &models.GitOpsSync{
+ MaxSyncFiles: 500,
+ MaxSyncTotalSize: 50 * 1024 * 1024,
+ MaxSyncBinarySize: 10 * 1024 * 1024,
+ }
+
+ maxFiles, maxTotalSize, maxBinarySize := svc.getEffectiveSyncLimits(ctx, sync)
+
+ require.Equal(t, 200, maxFiles)
+ require.Equal(t, int64(30*1024*1024), maxTotalSize)
+ require.Equal(t, int64(5*1024*1024), maxBinarySize)
+ })
+
+ t.Run("preserves tighter sync-specific limits", func(t *testing.T) {
+ sync := &models.GitOpsSync{
+ MaxSyncFiles: 75,
+ MaxSyncTotalSize: 8 * 1024 * 1024,
+ MaxSyncBinarySize: 2 * 1024 * 1024,
+ }
+
+ maxFiles, maxTotalSize, maxBinarySize := svc.getEffectiveSyncLimits(ctx, sync)
+
+ require.Equal(t, 75, maxFiles)
+ require.Equal(t, int64(8*1024*1024), maxTotalSize)
+ require.Equal(t, int64(2*1024*1024), maxBinarySize)
+ })
+
+ t.Run("treats zero as unlimited", func(t *testing.T) {
+ sync := &models.GitOpsSync{
+ MaxSyncFiles: 0,
+ MaxSyncTotalSize: 0,
+ MaxSyncBinarySize: 0,
+ }
+
+ maxFiles, maxTotalSize, maxBinarySize := svc.getEffectiveSyncLimits(ctx, sync)
+
+ require.Equal(t, 200, maxFiles)
+ require.Equal(t, int64(30*1024*1024), maxTotalSize)
+ require.Equal(t, int64(5*1024*1024), maxBinarySize)
+ })
+}
diff --git a/backend/internal/services/project_service.go b/backend/internal/services/project_service.go
index 8d7508462..a4728e355 100644
--- a/backend/internal/services/project_service.go
+++ b/backend/internal/services/project_service.go
@@ -594,6 +594,7 @@ func (s *ProjectService) GetProjectDetails(ctx context.Context, projectID string
// Enrich with details
s.enrichWithIncludeFiles(ctx, proj.Path, &resp)
+ s.enrichWithDirectoryFiles(ctx, proj.Path, &resp)
s.enrichWithGitOpsInfo(ctx, proj, &resp)
// Load compose project for service definitions
@@ -650,6 +651,114 @@ func (s *ProjectService) enrichWithIncludeFiles(ctx context.Context, projectPath
}
}
+func (s *ProjectService) enrichWithDirectoryFiles(ctx context.Context, projectPath string, resp *project.Details) {
+ if projectPath == "" {
+ return
+ }
+
+ // Build set of already-shown files to skip
+ shownFiles := map[string]bool{
+ ".env": true,
+ "compose.yaml": true,
+ "compose.yml": true,
+ "docker-compose.yaml": true,
+ "docker-compose.yml": true,
+ }
+ for _, inc := range resp.IncludeFiles {
+ shownFiles[inc.RelativePath] = true
+ }
+
+ var dirFiles []project.IncludeFile
+
+ root, err := os.OpenRoot(projectPath)
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to open project root for directory scan", "error", err, "path", projectPath)
+ resp.DirectoryFiles = dirFiles
+ return
+ }
+ defer func() { _ = root.Close() }()
+
+ err = s.collectDirectoryFiles(root, ".", projectPath, shownFiles, &dirFiles)
+
+ if err != nil {
+ slog.WarnContext(ctx, "Failed to scan project directory files", "error", err, "path", projectPath)
+ }
+
+ resp.DirectoryFiles = dirFiles
+}
+
+func (s *ProjectService) collectDirectoryFiles(
+ root *os.Root,
+ relDir string,
+ projectPath string,
+ shownFiles map[string]bool,
+ dirFiles *[]project.IncludeFile,
+) error {
+ dir, err := root.Open(relDir)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = dir.Close() }()
+
+ entries, err := dir.ReadDir(-1)
+ if err != nil {
+ return err
+ }
+
+ for _, entry := range entries {
+ relPath := entry.Name()
+ if relDir != "." {
+ relPath = filepath.Join(relDir, entry.Name())
+ }
+ if entry.Type()&os.ModeSymlink != 0 {
+ continue
+ }
+ if entry.IsDir() {
+ if entry.Name() == ".git" {
+ continue
+ }
+ if err := s.collectDirectoryFiles(root, relPath, projectPath, shownFiles, dirFiles); err != nil {
+ slog.Debug("Skipping unreadable project subdirectory", "relativePath", relPath, "error", err)
+ }
+ continue
+ }
+ if shownFiles[relPath] {
+ continue
+ }
+
+ info, err := entry.Info()
+ if err != nil || info.Size() > 1024*1024 {
+ continue
+ }
+
+ content, err := root.ReadFile(relPath)
+ if err != nil || isBinaryProjectFileContent(content) {
+ continue
+ }
+
+ *dirFiles = append(*dirFiles, project.IncludeFile{
+ Path: filepath.Join(projectPath, relPath),
+ RelativePath: relPath,
+ Content: string(content),
+ })
+ }
+
+ return nil
+}
+
+func isBinaryProjectFileContent(content []byte) bool {
+ checkSize := len(content)
+ if checkSize > 512 {
+ checkSize = 512
+ }
+ for _, b := range content[:checkSize] {
+ if b == 0 {
+ return true
+ }
+ }
+ return false
+}
+
func (s *ProjectService) enrichWithGitOpsInfo(ctx context.Context, proj *models.Project, resp *project.Details) {
if proj.GitOpsManagedBy != nil {
var sync models.GitOpsSync
diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.go
index beec2579c..ea1d1df20 100644
--- a/backend/internal/services/settings_service.go
+++ b/backend/internal/services/settings_service.go
@@ -160,6 +160,9 @@ func (s *SettingsService) getDefaultSettings() *models.Settings {
AccentColor: models.SettingVariable{Value: "oklch(0.606 0.25 292.717)"},
OledMode: models.SettingVariable{Value: "false"},
MaxImageUploadSize: models.SettingVariable{Value: "500"},
+ GitSyncMaxFiles: models.SettingVariable{Value: "500"},
+ GitSyncMaxTotalSizeMb: models.SettingVariable{Value: "50"},
+ GitSyncMaxBinarySizeMb: models.SettingVariable{Value: "10"},
EnvironmentHealthInterval: models.SettingVariable{Value: "0 */2 * * * *"},
DockerAPITimeout: models.SettingVariable{Value: "30"},
diff --git a/backend/internal/services/settings_service_test.go b/backend/internal/services/settings_service_test.go
index 98bbaabfa..e132dd467 100644
--- a/backend/internal/services/settings_service_test.go
+++ b/backend/internal/services/settings_service_test.go
@@ -44,7 +44,7 @@ func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {
require.Equal(t, count1, count2)
// Spot-check core and automation defaults exist with correct values
- for _, key := range []string{"authLocalEnabled", "projectsDirectory", "followProjectSymlinks", "autoUpdateExcludedContainers", "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyImage", "trivyNetwork", "trivySecurityOpts", "trivyPrivileged", "trivyPreserveCacheOnVolumePrune", "trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers"} {
+ for _, key := range []string{"authLocalEnabled", "projectsDirectory", "followProjectSymlinks", "autoUpdateExcludedContainers", "vulnerabilityScanEnabled", "vulnerabilityScanInterval", "trivyImage", "trivyNetwork", "trivySecurityOpts", "trivyPrivileged", "trivyPreserveCacheOnVolumePrune", "trivyResourceLimitsEnabled", "trivyCpuLimit", "trivyMemoryLimitMb", "trivyConcurrentScanContainers", "gitSyncMaxFiles", "gitSyncMaxTotalSizeMb", "gitSyncMaxBinarySizeMb"} {
var sv models.SettingVariable
err := svc.db.WithContext(ctx).Where("key = ?", key).First(&sv).Error
require.NoErrorf(t, err, "missing default key %s", key)
@@ -76,6 +76,12 @@ func TestSettingsService_EnsureDefaultSettings_Idempotent(t *testing.T) {
require.Equal(t, "0", sv.Value)
case "trivyConcurrentScanContainers":
require.Equal(t, "1", sv.Value)
+ case "gitSyncMaxFiles":
+ require.Equal(t, "500", sv.Value)
+ case "gitSyncMaxTotalSizeMb":
+ require.Equal(t, "50", sv.Value)
+ case "gitSyncMaxBinarySizeMb":
+ require.Equal(t, "10", sv.Value)
}
}
}
diff --git a/backend/pkg/gitutil/git.go b/backend/pkg/gitutil/git.go
index c14d74797..7057531c8 100644
--- a/backend/pkg/gitutil/git.go
+++ b/backend/pkg/gitutil/git.go
@@ -4,7 +4,10 @@ import (
"context"
"errors"
"fmt"
+ "io"
+ "io/fs"
"net"
+ nethttp "net/http"
"os"
"path/filepath"
"sort"
@@ -16,7 +19,7 @@ import (
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport"
- "github.com/go-git/go-git/v5/plumbing/transport/http"
+ githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/gofrs/flock"
gossh "golang.org/x/crypto/ssh"
@@ -56,7 +59,7 @@ func (c *Client) getAuth(config AuthConfig) (transport.AuthMethod, error) {
switch config.AuthType {
case "http":
if config.Token != "" {
- return &http.BasicAuth{
+ return &githttp.BasicAuth{
Username: config.Username,
Password: config.Token,
}, nil
@@ -499,3 +502,229 @@ func (c *Client) ReadFile(ctx context.Context, repoPath, filePath string) (strin
}
return string(content), nil
}
+
+// SyncFileInfo holds information about a file to be synced
+type SyncFileInfo struct {
+ RelativePath string // Path relative to the sync directory
+ Content []byte
+ Size int64
+ IsBinary bool
+}
+
+// DirectoryWalkResult holds the result of walking a directory for sync
+type DirectoryWalkResult struct {
+ Files []SyncFileInfo
+ TotalFiles int
+ TotalSize int64
+ SkippedBinaries int
+}
+
+type syncWalkLimits struct {
+ maxFiles int
+ maxTotalSize int64
+ maxBinarySize int64
+}
+
+// WalkDirectory walks the directory containing the compose file and returns all files.
+// It enforces limits on file count, total size, and skips large binary files.
+// The composePath is the path to the compose file within the repo - the directory
+// containing this file will be walked.
+func (c *Client) WalkDirectory(ctx context.Context, repoPath, composePath string,
+ maxFiles int, maxTotalSize, maxBinarySize int64) (*DirectoryWalkResult, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+
+ // Validate compose path
+ if err := ValidatePath(repoPath, composePath); err != nil {
+ return nil, fmt.Errorf("invalid compose path: %w", err)
+ }
+
+ // Get the directory containing the compose file
+ syncDir := filepath.Dir(filepath.Join(repoPath, composePath))
+
+ root, err := os.OpenRoot(syncDir)
+ if err != nil {
+ return nil, fmt.Errorf("sync directory not found: %w", err)
+ }
+ defer func() { _ = root.Close() }()
+
+ result := &DirectoryWalkResult{
+ Files: make([]SyncFileInfo, 0),
+ }
+ limits := syncWalkLimits{
+ maxFiles: maxFiles,
+ maxTotalSize: maxTotalSize,
+ maxBinarySize: maxBinarySize,
+ }
+
+ err = fs.WalkDir(root.FS(), ".", func(path string, d fs.DirEntry, walkErr error) error {
+ return c.walkSyncEntry(ctx, root, path, d, walkErr, result, limits)
+ })
+
+ if err != nil {
+ return nil, err
+ }
+
+ // Validate we found at least one file
+ if len(result.Files) == 0 {
+ return nil, fmt.Errorf("no files found in sync directory (directory may be empty or all files were skipped)")
+ }
+
+ return result, nil
+}
+
+func (c *Client) walkSyncEntry(
+ ctx context.Context,
+ root *os.Root,
+ path string,
+ d fs.DirEntry,
+ walkErr error,
+ result *DirectoryWalkResult,
+ limits syncWalkLimits,
+) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ if walkErr != nil {
+ return walkErr
+ }
+ if path == "." {
+ return nil
+ }
+ if d.Type()&os.ModeSymlink != 0 {
+ return nil
+ }
+ if d.IsDir() {
+ if d.Name() == ".git" {
+ return fs.SkipDir
+ }
+ return nil
+ }
+
+ return c.appendSyncFile(root, path, d, result, limits)
+}
+
+func (c *Client) appendSyncFile(root *os.Root, path string, d fs.DirEntry, result *DirectoryWalkResult, limits syncWalkLimits) error {
+ if limits.maxFiles > 0 && result.TotalFiles >= limits.maxFiles {
+ return fmt.Errorf("file count limit exceeded (max %d files)", limits.maxFiles)
+ }
+
+ if limits.maxBinarySize > 0 {
+ if info, err := d.Info(); err == nil && info.Size() > limits.maxBinarySize {
+ isBinary, err := c.isBinarySyncFile(root, path)
+ if err != nil {
+ return fmt.Errorf("failed to inspect file %s: %w", path, err)
+ }
+ if isBinary {
+ result.SkippedBinaries++
+ return nil
+ }
+ }
+ }
+
+ content, err := root.ReadFile(path)
+ if err != nil {
+ return fmt.Errorf("failed to read file %s: %w", path, err)
+ }
+
+ fileSize := int64(len(content))
+ isBinary := isBinaryContent(content)
+
+ if isBinary && limits.maxBinarySize > 0 && fileSize > limits.maxBinarySize {
+ result.SkippedBinaries++
+ return nil
+ }
+
+ if limits.maxTotalSize > 0 && result.TotalSize+fileSize > limits.maxTotalSize {
+ return fmt.Errorf("total size limit exceeded (max %d bytes)", limits.maxTotalSize)
+ }
+
+ result.Files = append(result.Files, SyncFileInfo{
+ RelativePath: path,
+ Content: content,
+ Size: fileSize,
+ IsBinary: isBinary,
+ })
+ result.TotalFiles++
+ result.TotalSize += fileSize
+
+ return nil
+}
+
+func (c *Client) isBinarySyncFile(root *os.Root, path string) (bool, error) {
+ file, err := root.Open(path)
+ if err != nil {
+ return false, err
+ }
+ defer func() { _ = file.Close() }()
+
+ buf := make([]byte, 512)
+ n, err := file.Read(buf)
+ if err != nil && !errors.Is(err, io.EOF) {
+ return false, err
+ }
+
+ return isBinaryContent(buf[:n]), nil
+}
+
+// isBinaryContent detects if content is binary using HTTP content type detection.
+// Returns true for binary content, false for text content.
+func isBinaryContent(content []byte) bool {
+ if len(content) == 0 {
+ return false
+ }
+
+ // Check first 512 bytes (or less if file is smaller)
+ checkSize := 512
+ if len(content) < checkSize {
+ checkSize = len(content)
+ }
+
+ // Use net/http's content type detection
+ contentType := detectContentType(content[:checkSize])
+
+ // Text types are not binary
+ if strings.HasPrefix(contentType, "text/") {
+ return false
+ }
+
+ // Common text-based application types
+ textAppTypes := []string{
+ "application/json",
+ "application/xml",
+ "application/javascript",
+ "application/x-yaml",
+ "application/yaml",
+ "application/toml",
+ "application/x-sh",
+ }
+ for _, t := range textAppTypes {
+ if strings.HasPrefix(contentType, t) {
+ return false
+ }
+ }
+
+ // For application/octet-stream, do additional null-byte check
+ // Text files rarely have null bytes, so their presence indicates binary
+ if contentType == "application/octet-stream" {
+ for _, b := range content[:checkSize] {
+ if b == 0 {
+ return true
+ }
+ }
+ // No null bytes found, likely a text file (Dockerfile, Makefile, etc.)
+ return false
+ }
+
+ // Everything else is considered binary
+ return strings.HasPrefix(contentType, "application/") ||
+ strings.HasPrefix(contentType, "image/") ||
+ strings.HasPrefix(contentType, "video/") ||
+ strings.HasPrefix(contentType, "audio/")
+}
+
+// detectContentType wraps nethttp.DetectContentType for easier testing
+func detectContentType(data []byte) string {
+ return nethttp.DetectContentType(data)
+}
diff --git a/backend/pkg/gitutil/git_test.go b/backend/pkg/gitutil/git_test.go
index cad662b0f..ad7a8cc60 100644
--- a/backend/pkg/gitutil/git_test.go
+++ b/backend/pkg/gitutil/git_test.go
@@ -1,6 +1,7 @@
package git
import (
+ "context"
"net"
"os"
"path/filepath"
@@ -294,6 +295,158 @@ func TestNewClient(t *testing.T) {
})
}
+func writeFile(t *testing.T, dir, name string, content []byte) {
+ t.Helper()
+ if err := os.WriteFile(filepath.Join(dir, name), content, 0644); err != nil {
+ t.Fatalf("failed to write file %s: %v", name, err)
+ }
+}
+
+func minimalCompose() []byte {
+ return []byte("services:\n test:\n image: alpine\n")
+}
+
+func TestWalkDirectory_BasicWalk(t *testing.T) {
+ tmpDir := t.TempDir()
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "file1.txt", []byte("hello world"))
+ writeFile(t, tmpDir, "file2.txt", []byte("another file"))
+
+ client := NewClient("")
+ result, err := client.WalkDirectory(context.Background(), tmpDir, "compose.yaml", 0, 0, 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if result.TotalFiles != 3 {
+ t.Errorf("expected 3 files, got %d", result.TotalFiles)
+ }
+ if len(result.Files) != 3 {
+ t.Errorf("expected 3 entries in Files, got %d", len(result.Files))
+ }
+}
+
+func TestWalkDirectory_MaxFilesLimit(t *testing.T) {
+ tmpDir := t.TempDir()
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "a.txt", []byte("a"))
+ writeFile(t, tmpDir, "b.txt", []byte("b"))
+ writeFile(t, tmpDir, "c.txt", []byte("c"))
+ writeFile(t, tmpDir, "d.txt", []byte("d"))
+
+ client := NewClient("")
+ _, err := client.WalkDirectory(context.Background(), tmpDir, "compose.yaml", 3, 0, 0)
+ if err == nil {
+ t.Fatal("expected error for file count limit, got nil")
+ }
+ if !strings.Contains(err.Error(), "file count limit exceeded") {
+ t.Errorf("expected 'file count limit exceeded' error, got: %v", err)
+ }
+}
+
+func TestWalkDirectory_MaxFilesUnlimited(t *testing.T) {
+ tmpDir := t.TempDir()
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "a.txt", []byte("a"))
+ writeFile(t, tmpDir, "b.txt", []byte("b"))
+ writeFile(t, tmpDir, "c.txt", []byte("c"))
+ writeFile(t, tmpDir, "d.txt", []byte("d"))
+
+ client := NewClient("")
+ result, err := client.WalkDirectory(context.Background(), tmpDir, "compose.yaml", 0, 0, 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if result.TotalFiles != 5 {
+ t.Errorf("expected 5 files, got %d", result.TotalFiles)
+ }
+}
+
+func TestWalkDirectory_MaxTotalSizeLimit(t *testing.T) {
+ tmpDir := t.TempDir()
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 40)))
+ writeFile(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 40)))
+
+ client := NewClient("")
+ _, err := client.WalkDirectory(context.Background(), tmpDir, "compose.yaml", 0, 50, 0)
+ if err == nil {
+ t.Fatal("expected error for total size limit, got nil")
+ }
+ if !strings.Contains(err.Error(), "total size limit exceeded") {
+ t.Errorf("expected 'total size limit exceeded' error, got: %v", err)
+ }
+}
+
+func TestWalkDirectory_MaxTotalSizeUnlimited(t *testing.T) {
+ tmpDir := t.TempDir()
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "big1.txt", []byte(strings.Repeat("x", 500)))
+ writeFile(t, tmpDir, "big2.txt", []byte(strings.Repeat("y", 500)))
+
+ client := NewClient("")
+ result, err := client.WalkDirectory(context.Background(), tmpDir, "compose.yaml", 0, 0, 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if result.TotalFiles != 3 {
+ t.Errorf("expected 3 files, got %d", result.TotalFiles)
+ }
+}
+
+func TestWalkDirectory_MaxBinarySizeSkips(t *testing.T) {
+ tmpDir := t.TempDir()
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ // Binary content: null bytes cause isBinaryContent to return true
+ binaryContent := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13}
+ writeFile(t, tmpDir, "data.bin", binaryContent)
+
+ client := NewClient("")
+ result, err := client.WalkDirectory(context.Background(), tmpDir, "compose.yaml", 0, 0, 5)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if result.SkippedBinaries == 0 {
+ t.Error("expected at least one skipped binary file, got 0")
+ }
+}
+
+func TestWalkDirectory_MaxBinarySizeUnlimited(t *testing.T) {
+ tmpDir := t.TempDir()
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ binaryContent := []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13}
+ writeFile(t, tmpDir, "data.bin", binaryContent)
+
+ client := NewClient("")
+ result, err := client.WalkDirectory(context.Background(), tmpDir, "compose.yaml", 0, 0, 0)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if result.SkippedBinaries != 0 {
+ t.Errorf("expected no skipped binaries with unlimited size, got %d", result.SkippedBinaries)
+ }
+ if result.TotalFiles != 2 {
+ t.Errorf("expected 2 files (compose + binary), got %d", result.TotalFiles)
+ }
+}
+
+func TestWalkDirectory_LargeTextFileNotSkippedByBinaryLimit(t *testing.T) {
+ tmpDir := t.TempDir()
+ writeFile(t, tmpDir, "compose.yaml", minimalCompose())
+ writeFile(t, tmpDir, "notes.txt", []byte(strings.Repeat("plain text\n", 32)))
+
+ client := NewClient("")
+ result, err := client.WalkDirectory(context.Background(), tmpDir, "compose.yaml", 0, 0, 16)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if result.SkippedBinaries != 0 {
+ t.Errorf("expected no skipped binaries for large text file, got %d", result.SkippedBinaries)
+ }
+ if result.TotalFiles != 2 {
+ t.Errorf("expected 2 files (compose + text), got %d", result.TotalFiles)
+ }
+}
+
// generateTestPublicKey creates a test ED25519 public key for testing
func generateTestPublicKey(t *testing.T) gossh.PublicKey {
t.Helper()
diff --git a/backend/pkg/projects/fs_writer.go b/backend/pkg/projects/fs_writer.go
index 7517fcf3f..2dc724df8 100644
--- a/backend/pkg/projects/fs_writer.go
+++ b/backend/pkg/projects/fs_writer.go
@@ -3,6 +3,7 @@ package projects
import (
"errors"
"fmt"
+ "log/slog"
"os"
"path/filepath"
"strings"
@@ -204,3 +205,147 @@ func WriteFileWithPerm(filePath, content string, perm os.FileMode) error {
return nil
}
+
+// SyncFile represents a file to be written during directory sync
+type SyncFile struct {
+ RelativePath string // Path relative to the project directory
+ Content []byte
+}
+
+// WriteSyncedDirectory writes multiple files to a project directory.
+// It validates all paths are within the project directory and creates
+// subdirectories as needed. Returns the list of written file paths.
+func WriteSyncedDirectory(projectsRoot, projectPath string, files []SyncFile) ([]string, error) {
+ // Security: Validate projectPath is within projectsRoot
+ rootAbs, err := filepath.Abs(projectsRoot)
+ if err != nil {
+ return nil, fmt.Errorf("failed to resolve projects root: %w", err)
+ }
+ rootAbs = filepath.Clean(rootAbs)
+
+ projectAbs, err := filepath.Abs(projectPath)
+ if err != nil {
+ return nil, fmt.Errorf("failed to resolve project path: %w", err)
+ }
+ projectAbs = filepath.Clean(projectAbs)
+
+ if !IsSafeSubdirectory(rootAbs, projectAbs) {
+ return nil, fmt.Errorf("project path is outside projects root")
+ }
+
+ // Ensure project directory exists
+ if err := os.MkdirAll(projectAbs, pkgutils.DirPerm); err != nil {
+ return nil, fmt.Errorf("failed to create project directory: %w", err)
+ }
+
+ writtenPaths := make([]string, 0, len(files))
+
+ for _, file := range files {
+ // Validate relative path doesn't escape project directory
+ targetPath := filepath.Join(projectAbs, file.RelativePath)
+ targetPathClean := filepath.Clean(targetPath)
+
+ if !IsSafeSubdirectory(projectAbs, targetPathClean) {
+ return nil, fmt.Errorf("file path %s would escape project directory", file.RelativePath)
+ }
+
+ // Create parent directories
+ parentDir := filepath.Dir(targetPathClean)
+ if err := os.MkdirAll(parentDir, pkgutils.DirPerm); err != nil {
+ return nil, fmt.Errorf("failed to create directory for %s: %w", file.RelativePath, err)
+ }
+
+ // Write the file
+ if err := os.WriteFile(targetPathClean, file.Content, pkgutils.FilePerm); err != nil {
+ return nil, fmt.Errorf("failed to write file %s: %w", file.RelativePath, err)
+ }
+
+ writtenPaths = append(writtenPaths, file.RelativePath)
+ }
+
+ return writtenPaths, nil
+}
+
+// CleanupRemovedFiles deletes files that were in the old sync but are not in the new sync.
+// It only removes files that were previously synced (tracked in oldFiles).
+// Empty directories are removed after file deletion.
+// This is a best-effort operation - errors are logged but don't cause failure.
+func CleanupRemovedFiles(projectsRoot, projectPath string, oldFiles, newFiles []string) error {
+ // Security: Validate projectPath is within projectsRoot
+ rootAbs, err := filepath.Abs(projectsRoot)
+ if err != nil {
+ return fmt.Errorf("failed to resolve projects root: %w", err)
+ }
+ rootAbs = filepath.Clean(rootAbs)
+
+ projectAbs, err := filepath.Abs(projectPath)
+ if err != nil {
+ return fmt.Errorf("failed to resolve project path: %w", err)
+ }
+ projectAbs = filepath.Clean(projectAbs)
+
+ if !IsSafeSubdirectory(rootAbs, projectAbs) {
+ return fmt.Errorf("project path is outside projects root")
+ }
+
+ // Build set of new files for quick lookup
+ newFileSet := make(map[string]bool, len(newFiles))
+ for _, f := range newFiles {
+ newFileSet[f] = true
+ }
+
+ // Track directories that may need cleanup
+ dirsToCheck := make(map[string]bool)
+
+ // Delete files that are in old but not in new
+ for _, oldFile := range oldFiles {
+ if newFileSet[oldFile] {
+ continue // File still exists in new sync
+ }
+
+ targetPath := filepath.Join(projectAbs, oldFile)
+ targetPathClean := filepath.Clean(targetPath)
+
+ // Security check
+ if !IsSafeSubdirectory(projectAbs, targetPathClean) {
+ continue // Skip files that would be outside project
+ }
+
+ // Delete the file (best effort)
+ if err := os.Remove(targetPathClean); err != nil {
+ if !os.IsNotExist(err) {
+ // Log but continue - this is best effort
+ slog.Warn("Failed to remove old synced file", "file", oldFile, "error", err)
+ }
+ }
+
+ // Track parent directory for potential cleanup
+ parentDir := filepath.Dir(targetPathClean)
+ if parentDir != projectAbs {
+ dirsToCheck[parentDir] = true
+ }
+ }
+
+ // Clean up empty directories (best effort, deepest first)
+ for dir := range dirsToCheck {
+ cleanupEmptyDirs(projectAbs, dir)
+ }
+
+ return nil
+}
+
+// cleanupEmptyDirs removes empty directories starting from the given path
+// up to (but not including) the project root.
+func cleanupEmptyDirs(projectRoot, startDir string) {
+ current := startDir
+ for current != projectRoot && IsSafeSubdirectory(projectRoot, current) {
+ // Try to remove the directory (will fail if not empty)
+ err := os.Remove(current)
+ if err != nil {
+ break // Directory not empty or other error
+ }
+
+ // Move to parent directory
+ current = filepath.Dir(current)
+ }
+}
diff --git a/backend/resources/migrations/postgres/042_add_directory_sync.down.sql b/backend/resources/migrations/postgres/042_add_directory_sync.down.sql
new file mode 100644
index 000000000..7e1558e96
--- /dev/null
+++ b/backend/resources/migrations/postgres/042_add_directory_sync.down.sql
@@ -0,0 +1,9 @@
+-- Remove directory sync columns from gitops_syncs table
+
+DROP INDEX IF EXISTS idx_gitops_syncs_sync_directory;
+
+ALTER TABLE gitops_syncs DROP COLUMN IF EXISTS max_sync_binary_size;
+ALTER TABLE gitops_syncs DROP COLUMN IF EXISTS max_sync_total_size;
+ALTER TABLE gitops_syncs DROP COLUMN IF EXISTS max_sync_files;
+ALTER TABLE gitops_syncs DROP COLUMN IF EXISTS synced_files;
+ALTER TABLE gitops_syncs DROP COLUMN IF EXISTS sync_directory;
diff --git a/backend/resources/migrations/postgres/042_add_directory_sync.up.sql b/backend/resources/migrations/postgres/042_add_directory_sync.up.sql
new file mode 100644
index 000000000..5a52d5212
--- /dev/null
+++ b/backend/resources/migrations/postgres/042_add_directory_sync.up.sql
@@ -0,0 +1,14 @@
+-- Add directory sync support to gitops_syncs table
+-- sync_directory: when true, syncs entire directory containing compose file (default: true)
+-- synced_files: JSON array of file paths that were synced (for cleanup on updates)
+-- max_sync_files: maximum number of files to sync (0 = unlimited)
+-- max_sync_total_size: maximum total size in bytes (0 = unlimited)
+-- max_sync_binary_size: maximum binary file size in bytes (0 = unlimited)
+
+ALTER TABLE gitops_syncs ADD COLUMN sync_directory BOOLEAN NOT NULL DEFAULT true;
+ALTER TABLE gitops_syncs ADD COLUMN synced_files TEXT;
+ALTER TABLE gitops_syncs ADD COLUMN max_sync_files INTEGER NOT NULL DEFAULT 500;
+ALTER TABLE gitops_syncs ADD COLUMN max_sync_total_size BIGINT NOT NULL DEFAULT 52428800;
+ALTER TABLE gitops_syncs ADD COLUMN max_sync_binary_size BIGINT NOT NULL DEFAULT 10485760;
+
+CREATE INDEX IF NOT EXISTS idx_gitops_syncs_sync_directory ON gitops_syncs(sync_directory);
diff --git a/backend/resources/migrations/sqlite/042_add_directory_sync.down.sql b/backend/resources/migrations/sqlite/042_add_directory_sync.down.sql
new file mode 100644
index 000000000..5e12cefdf
--- /dev/null
+++ b/backend/resources/migrations/sqlite/042_add_directory_sync.down.sql
@@ -0,0 +1,55 @@
+-- Remove directory sync columns from gitops_syncs table
+-- SQLite doesn't support DROP COLUMN in older versions, so we recreate the table
+
+DROP INDEX IF EXISTS idx_gitops_syncs_sync_directory;
+
+CREATE TABLE gitops_syncs_backup AS SELECT
+ id,
+ name,
+ environment_id,
+ repository_id,
+ branch,
+ compose_path,
+ project_name,
+ project_id,
+ auto_sync,
+ sync_interval,
+ last_sync_at,
+ last_sync_status,
+ last_sync_error,
+ last_sync_commit,
+ created_at,
+ updated_at
+FROM gitops_syncs;
+
+DROP TABLE gitops_syncs;
+
+CREATE TABLE gitops_syncs (
+ id TEXT PRIMARY KEY,
+ name TEXT NOT NULL,
+ environment_id TEXT NOT NULL,
+ repository_id TEXT NOT NULL,
+ branch TEXT NOT NULL,
+ compose_path TEXT NOT NULL,
+ project_name TEXT NOT NULL,
+ project_id TEXT,
+ auto_sync INTEGER NOT NULL DEFAULT 0,
+ sync_interval INTEGER NOT NULL DEFAULT 60,
+ last_sync_at DATETIME,
+ last_sync_status TEXT,
+ last_sync_error TEXT,
+ last_sync_commit TEXT,
+ created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ updated_at DATETIME,
+ FOREIGN KEY (environment_id) REFERENCES environments(id) ON DELETE CASCADE,
+ FOREIGN KEY (repository_id) REFERENCES git_repositories(id) ON DELETE CASCADE,
+ FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE SET NULL
+);
+
+INSERT INTO gitops_syncs SELECT * FROM gitops_syncs_backup;
+DROP TABLE gitops_syncs_backup;
+
+CREATE INDEX IF NOT EXISTS idx_gitops_syncs_environment_id ON gitops_syncs(environment_id);
+CREATE INDEX IF NOT EXISTS idx_gitops_syncs_repository_id ON gitops_syncs(repository_id);
+CREATE INDEX IF NOT EXISTS idx_gitops_syncs_project_id ON gitops_syncs(project_id);
+CREATE INDEX IF NOT EXISTS idx_gitops_syncs_auto_sync ON gitops_syncs(auto_sync);
diff --git a/backend/resources/migrations/sqlite/042_add_directory_sync.up.sql b/backend/resources/migrations/sqlite/042_add_directory_sync.up.sql
new file mode 100644
index 000000000..dac5166aa
--- /dev/null
+++ b/backend/resources/migrations/sqlite/042_add_directory_sync.up.sql
@@ -0,0 +1,14 @@
+-- Add directory sync support to gitops_syncs table
+-- sync_directory: when true, syncs entire directory containing compose file (default: true)
+-- synced_files: JSON array of file paths that were synced (for cleanup on updates)
+-- max_sync_files: maximum number of files to sync (0 = unlimited)
+-- max_sync_total_size: maximum total size in bytes (0 = unlimited)
+-- max_sync_binary_size: maximum binary file size in bytes (0 = unlimited)
+
+ALTER TABLE gitops_syncs ADD COLUMN sync_directory INTEGER NOT NULL DEFAULT 1;
+ALTER TABLE gitops_syncs ADD COLUMN synced_files TEXT;
+ALTER TABLE gitops_syncs ADD COLUMN max_sync_files INTEGER NOT NULL DEFAULT 500;
+ALTER TABLE gitops_syncs ADD COLUMN max_sync_total_size INTEGER NOT NULL DEFAULT 52428800;
+ALTER TABLE gitops_syncs ADD COLUMN max_sync_binary_size INTEGER NOT NULL DEFAULT 10485760;
+
+CREATE INDEX IF NOT EXISTS idx_gitops_syncs_sync_directory ON gitops_syncs(sync_directory);
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 6b8fad51f..db2cc81ba 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -1414,6 +1414,8 @@
"git_sync_repository": "Git Repository",
"git_sync_branch": "Branch",
"git_sync_compose_path": "Compose File Path",
+ "git_sync_sync_files": "Sync Files",
+ "git_sync_sync_files_description": "Copy sibling files from the compose directory into the managed project. When disabled, Arcane only syncs the selected compose file.",
"git_sync_auto_sync": "Auto Sync",
"git_sync_sync_interval": "Sync Interval (minutes)",
"git_sync_last_sync": "Last Sync",
@@ -1456,6 +1458,14 @@
"git_manage_syncs": "Manage Git Syncs",
"git_from_git_repo": "From Git Repo",
"git_sync_from_git": "Sync from Git",
+ "git_sync_file_limits_title": "Git Sync File Limits",
+ "git_sync_file_limits_description": "Apply environment-wide caps when Arcane copies repository files during Git syncs. Use 0 to remove a cap.",
+ "git_sync_max_files_label": "Max Synced Files",
+ "git_sync_max_files_help": "Maximum number of files copied from a repository per sync. Set 0 to disable the environment cap.",
+ "git_sync_max_total_size_label": "Max Total Size (MB)",
+ "git_sync_max_total_size_help": "Maximum combined size of synced files per sync. Set 0 to disable the environment cap.",
+ "git_sync_max_binary_size_label": "Max Binary Size (MB)",
+ "git_sync_max_binary_size_help": "Maximum size for a single binary file copied during sync. Set 0 to disable the environment cap.",
"common_syncing": "Syncing...",
"_comment_customize_variables": "=== CUSTOMIZATION - VARIABLES ===",
"variables_title": "Variables",
diff --git a/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte b/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte
index 76f10b42c..0bea510ef 100644
--- a/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte
+++ b/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte
@@ -33,6 +33,7 @@
repositoryId: z.string().min(1, m.common_required()),
branch: z.string().min(1, m.common_required()),
composePath: z.string().min(1, m.common_required()),
+ syncDirectory: z.boolean().default(true),
autoSync: z.boolean().default(true),
syncInterval: z.number().min(1).default(5)
});
@@ -42,6 +43,7 @@
repositoryId: open && syncToEdit ? syncToEdit.repositoryId : '',
branch: open && syncToEdit ? syncToEdit.branch : 'main',
composePath: open && syncToEdit ? syncToEdit.composePath : 'docker-compose.yml',
+ syncDirectory: open && syncToEdit ? (syncToEdit.syncDirectory ?? true) : true,
autoSync: open && syncToEdit ? (syncToEdit.autoSync ?? true) : true,
syncInterval: open && syncToEdit ? (syncToEdit.syncInterval ?? 5) : 5
});
@@ -104,6 +106,7 @@
branch: data.branch,
composePath: data.composePath,
projectName: data.name,
+ syncDirectory: data.syncDirectory,
autoSync: data.autoSync,
syncInterval: data.syncInterval
};
@@ -220,6 +223,14 @@
{/if}
</div>
+ <SwitchWithLabel
+ id="syncDirectorySwitch"
+ label={m.git_sync_sync_files()}
+ description={m.git_sync_sync_files_description()}
+ error={$inputs.syncDirectory.error}
+ bind:checked={$inputs.syncDirectory.value}
+ />
+
<SwitchWithLabel
id="autoSyncSwitch"
label={m.git_sync_auto_sync()}
diff --git a/frontend/src/lib/types/gitops.type.ts b/frontend/src/lib/types/gitops.type.ts
index e5302a91e..bfe43cdbe 100644
--- a/frontend/src/lib/types/gitops.type.ts
+++ b/frontend/src/lib/types/gitops.type.ts
@@ -43,6 +43,10 @@ export interface GitOpsSyncCreateDto {
projectName?: string;
autoSync?: boolean;
syncInterval?: number;
+ syncDirectory?: boolean;
+ maxSyncFiles?: number;
+ maxSyncTotalSize?: number;
+ maxSyncBinarySize?: number;
}
export interface GitOpsSyncUpdateDto {
@@ -53,6 +57,10 @@ export interface GitOpsSyncUpdateDto {
projectName?: string;
autoSync?: boolean;
syncInterval?: number;
+ syncDirectory?: boolean;
+ maxSyncFiles?: number;
+ maxSyncTotalSize?: number;
+ maxSyncBinarySize?: number;
}
export interface GitOpsSync {
@@ -67,6 +75,11 @@ export interface GitOpsSync {
projectId?: string;
autoSync: boolean;
syncInterval: number;
+ syncDirectory: boolean;
+ syncedFiles?: string;
+ maxSyncFiles: number;
+ maxSyncTotalSize: number;
+ maxSyncBinarySize: number;
lastSyncAt?: string;
lastSyncStatus?: string;
lastSyncError?: string;
diff --git a/frontend/src/lib/types/project.type.ts b/frontend/src/lib/types/project.type.ts
index 03ad24512..1533e2d68 100644
--- a/frontend/src/lib/types/project.type.ts
+++ b/frontend/src/lib/types/project.type.ts
@@ -85,6 +85,7 @@ export interface Project {
composeContent?: string;
envContent?: string;
includeFiles?: IncludeFile[];
+ directoryFiles?: IncludeFile[];
}
export interface ProjectStatusCounts {
diff --git a/frontend/src/lib/types/settings.type.ts b/frontend/src/lib/types/settings.type.ts
index 95aff123e..ec5641d40 100644
--- a/frontend/src/lib/types/settings.type.ts
+++ b/frontend/src/lib/types/settings.type.ts
@@ -28,6 +28,9 @@ export type Settings = {
autoHealMaxRestarts?: number;
autoHealRestartWindow?: number;
maxImageUploadSize: number;
+ gitSyncMaxFiles: number;
+ gitSyncMaxTotalSizeMb: number;
+ gitSyncMaxBinarySizeMb: number;
baseServerUrl: string;
enableGravatar: boolean;
uiConfigDisabled: boolean;
diff --git a/frontend/src/routes/(app)/environments/[id]/+page.svelte b/frontend/src/routes/(app)/environments/[id]/+page.svelte
index 604b45018..0d9ab89d9 100644
--- a/frontend/src/routes/(app)/environments/[id]/+page.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/+page.svelte
@@ -156,6 +156,9 @@
projectsDirectory: z.string(),
diskUsagePath: z.string(),
maxImageUploadSize: z.coerce.number(),
+ gitSyncMaxFiles: z.coerce.number().int().nonnegative(),
+ gitSyncMaxTotalSizeMb: z.coerce.number().int().nonnegative(),
+ gitSyncMaxBinarySizeMb: z.coerce.number().int().nonnegative(),
baseServerUrl: z.string(),
scheduledPruneEnabled: z.boolean(),
scheduledPruneContainers: z.boolean(),
@@ -195,6 +198,9 @@
projectsDirectory: settings?.projectsDirectory || '/app/data/projects',
diskUsagePath: settings?.diskUsagePath || '/app/data/projects',
maxImageUploadSize: settings?.maxImageUploadSize || 500,
+ gitSyncMaxFiles: settings?.gitSyncMaxFiles ?? 500,
+ gitSyncMaxTotalSizeMb: settings?.gitSyncMaxTotalSizeMb ?? 50,
+ gitSyncMaxBinarySizeMb: settings?.gitSyncMaxBinarySizeMb ?? 10,
baseServerUrl: settings?.baseServerUrl || 'http://localhost',
scheduledPruneEnabled: settings?.scheduledPruneEnabled ?? false,
scheduledPruneContainers: settings?.scheduledPruneContainers ?? true,
@@ -241,6 +247,9 @@
projectsDirectory: formData.projectsDirectory,
diskUsagePath: formData.diskUsagePath,
maxImageUploadSize: formData.maxImageUploadSize,
+ gitSyncMaxFiles: formData.gitSyncMaxFiles,
+ gitSyncMaxTotalSizeMb: formData.gitSyncMaxTotalSizeMb,
+ gitSyncMaxBinarySizeMb: formData.gitSyncMaxBinarySizeMb,
baseServerUrl: formData.baseServerUrl,
scheduledPruneEnabled: formData.scheduledPruneEnabled,
scheduledPruneContainers: formData.scheduledPruneContainers,
diff --git a/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte
index f7de6d083..a247a13ee 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte
@@ -57,6 +57,38 @@
helpText={m.docker_max_upload_size_description()}
/>
</div>
+ <div class="space-y-4 rounded-lg border p-4 sm:col-span-2">
+ <div class="space-y-0.5">
+ <h3 class="text-sm font-medium">{m.git_sync_file_limits_title()}</h3>
+ <div class="text-muted-foreground text-xs">{m.git_sync_file_limits_description()}</div>
+ </div>
+ <div class="grid gap-4 sm:grid-cols-3">
+ <TextInputWithLabel
+ id="git-sync-max-files"
+ type="number"
+ label={m.git_sync_max_files_label()}
+ bind:value={$formInputs.gitSyncMaxFiles.value}
+ error={$formInputs.gitSyncMaxFiles.error}
+ helpText={m.git_sync_max_files_help()}
+ />
+ <TextInputWithLabel
+ id="git-sync-max-total-size"
+ type="number"
+ label={m.git_sync_max_total_size_label()}
+ bind:value={$formInputs.gitSyncMaxTotalSizeMb.value}
+ error={$formInputs.gitSyncMaxTotalSizeMb.error}
+ helpText={m.git_sync_max_total_size_help()}
+ />
+ <TextInputWithLabel
+ id="git-sync-max-binary-size"
+ type="number"
+ label={m.git_sync_max_binary_size_label()}
+ bind:value={$formInputs.gitSyncMaxBinarySizeMb.value}
+ error={$formInputs.gitSyncMaxBinarySizeMb.error}
+ helpText={m.git_sync_max_binary_size_help()}
+ />
+ </div>
+ </div>
<div class="space-y-4 rounded-lg border p-4 sm:col-span-2">
<div class="flex items-center justify-between gap-4">
<div class="space-y-0.5">
diff --git a/frontend/src/routes/(app)/projects/[projectId]/+page.svelte b/frontend/src/routes/(app)/projects/[projectId]/+page.svelte
index c7b3493f5..909401953 100644
--- a/frontend/src/routes/(app)/projects/[projectId]/+page.svelte
+++ b/frontend/src/routes/(app)/projects/[projectId]/+page.svelte
@@ -304,9 +304,10 @@
autoScrollStackLogs = cur.autoScroll ?? defaultComposeUIPrefs.autoScroll;
selectedFile = cur.selectedFile ?? defaultComposeUIPrefs.selectedFile ?? 'compose';
- // Auto-detect layout mode based on includeFiles
+ // Auto-detect layout mode based on includeFiles or directoryFiles
const hasIncludes = project?.includeFiles && project.includeFiles.length > 0;
- const defaultMode = hasIncludes ? 'tree' : 'classic';
+ const hasDirectoryFiles = project?.directoryFiles && project.directoryFiles.length > 0;
+ const defaultMode = hasIncludes || hasDirectoryFiles ? 'tree' : 'classic';
layoutMode = cur.layoutMode ?? defaultMode;
});
@@ -645,7 +646,7 @@
{#if isTablet.current}
<div class="flex h-full min-h-0 flex-col gap-4">
<Card.Root class="flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
- <Card.Header icon={FileTextIcon} class="shrink-0 items-center">
+ <Card.Header icon={FileTextIcon} class="flex-shrink-0 items-center">
<Card.Title>
<h2>{m.project_files()}</h2>
</Card.Title>
@@ -687,6 +688,20 @@
{/each}
</TreeView.Folder>
{/if}
+
+ {#if project?.directoryFiles && project.directoryFiles.length > 0}
+ {#each project.directoryFiles as dirFile (dirFile.relativePath)}
+ <TreeView.File
+ name={dirFile.relativePath}
+ onclick={() => (selectedFile = `dir:${dirFile.relativePath}`)}
+ class={selectedFile === `dir:${dirFile.relativePath}` ? 'bg-accent' : ''}
+ >
+ {#snippet icon()}
+ <FileTextIcon class="text-muted-foreground size-4" />
+ {/snippet}
+ </TreeView.File>
+ {/each}
+ {/if}
</TreeView.Root>
</Card.Content>
</Card.Root>
@@ -700,12 +715,6 @@
bind:value={$inputs.composeContent.value}
error={$inputs.composeContent.error ?? undefined}
readOnly={!canEditCompose}
- bind:hasErrors={composeHasErrors}
- bind:validationReady={composeValidationReady}
- fileId={`project:${projectId}:compose`}
- originalValue={originalComposeContent}
- enableDiff={true}
- editorContext={codeEditorContext}
/>
{:else if selectedFile === 'env'}
<CodePanel
@@ -714,14 +723,19 @@
language="env"
bind:value={$inputs.envContent.value}
error={$inputs.envContent.error ?? undefined}
- readOnly={!canEditEnv}
- bind:hasErrors={envHasErrors}
- bind:validationReady={envValidationReady}
- fileId={`project:${projectId}:env`}
- originalValue={originalEnvContent}
- enableDiff={true}
- editorContext={codeEditorContext}
/>
+ {:else if selectedFile.startsWith('dir:')}
+ {@const dirRelPath = selectedFile.slice(4)}
+ {@const dirFile = project?.directoryFiles?.find((f) => f.relativePath === dirRelPath)}
+ {#if dirFile}
+ <CodePanel
+ open={true}
+ title={dirFile.relativePath}
+ language="yaml"
+ value={dirFile.content}
+ readOnly={true}
+ />
+ {/if}
{:else}
{@const includeFile = project?.includeFiles?.find((f) => f.relativePath === selectedFile)}
{#if includeFile}
@@ -730,12 +744,6 @@
title={includeFile.relativePath}
language="yaml"
bind:value={includeFilesState[includeFile.relativePath]}
- bind:hasErrors={includeFilesHasErrors[includeFile.relativePath]}
- bind:validationReady={includeFilesValidationReady[includeFile.relativePath]}
- fileId={`project:${projectId}:include:${includeFile.relativePath}`}
- originalValue={originalIncludeFiles[includeFile.relativePath]}
- enableDiff={true}
- editorContext={codeEditorContext}
/>
{/if}
{/if}
@@ -798,6 +806,20 @@
{/each}
</TreeView.Folder>
{/if}
+
+ {#if project?.directoryFiles && project.directoryFiles.length > 0}
+ {#each project.directoryFiles as dirFile (dirFile.relativePath)}
+ <TreeView.File
+ name={dirFile.relativePath}
+ onclick={() => (selectedFile = `dir:${dirFile.relativePath}`)}
+ class={selectedFile === `dir:${dirFile.relativePath}` ? 'bg-accent' : ''}
+ >
+ {#snippet icon()}
+ <FileTextIcon class="text-muted-foreground size-4" />
+ {/snippet}
+ </TreeView.File>
+ {/each}
+ {/if}
</TreeView.Root>
</Card.Content>
</Card.Root>
@@ -835,6 +857,18 @@
enableDiff={true}
editorContext={codeEditorContext}
/>
+ {:else if selectedFile.startsWith('dir:')}
+ {@const dirRelPath = selectedFile.slice(4)}
+ {@const dirFile = project?.directoryFiles?.find((f) => f.relativePath === dirRelPath)}
+ {#if dirFile}
+ <CodePanel
+ open={true}
+ title={dirFile.relativePath}
+ language="yaml"
+ value={dirFile.content}
+ readOnly={true}
+ />
+ {/if}
{:else}
{@const includeFile = project?.includeFiles?.find((f) => f.relativePath === selectedFile)}
{#if includeFile}
diff --git a/tests/setup/gitops.setup.ts b/tests/setup/gitops.setup.ts
index 2c1485d04..575489199 100644
--- a/tests/setup/gitops.setup.ts
+++ b/tests/setup/gitops.setup.ts
@@ -127,7 +127,7 @@ setup('create gitops sync in arcane', async ({ page }) => {
}
// Disable auto-sync for tests
- const autoSyncSwitch = dialog.getByRole('switch');
+ const autoSyncSwitch = dialog.locator('#autoSyncSwitch');
if ((await autoSyncSwitch.count()) > 0) {
const isChecked = await autoSyncSwitch.getAttribute('data-state');
if (isChecked === 'checked') {
diff --git a/types/gitops/gitops.go b/types/gitops/gitops.go
index b18d1cfbd..56442e1bf 100644
--- a/types/gitops/gitops.go
+++ b/types/gitops/gitops.go
@@ -112,6 +112,37 @@ type GitOpsSync struct {
// Required: true
SyncInterval int `json:"syncInterval"`
+ // SyncDirectory indicates if the entire directory containing the compose file should be synced.
+ // When true, all files in the compose file's directory (and subdirectories) are synced.
+ // When false, only the compose file itself is synced.
+ //
+ // Required: true
+ SyncDirectory bool `json:"syncDirectory"`
+
+ // SyncedFiles is a JSON-encoded list of file paths that were synced in the last successful sync.
+ // Only populated when SyncDirectory is true. Parse as JSON array of strings.
+ //
+ // Required: false
+ SyncedFiles *string `json:"syncedFiles,omitempty"`
+
+ // MaxSyncFiles is the maximum number of files to sync.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: true
+ MaxSyncFiles int `json:"maxSyncFiles"`
+
+ // MaxSyncTotalSize is the maximum total size in bytes for all synced files.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: true
+ MaxSyncTotalSize int64 `json:"maxSyncTotalSize"`
+
+ // MaxSyncBinarySize is the maximum size in bytes for individual binary files.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: true
+ MaxSyncBinarySize int64 `json:"maxSyncBinarySize"`
+
// LastSyncAt is the date and time of the last successful sync.
//
// Required: false
@@ -298,6 +329,34 @@ type CreateSyncRequest struct {
//
// Required: false
SyncInterval *int `json:"syncInterval,omitempty"`
+
+ // SyncDirectory indicates if the entire directory containing the compose file should be synced.
+ // When true (default), all files in the compose file's directory are synced.
+ // When false, only the compose file itself is synced.
+ //
+ // Required: false
+ SyncDirectory *bool `json:"syncDirectory,omitempty"`
+
+ // MaxSyncFiles is the maximum number of files to sync.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ // Default: 500
+ //
+ // Required: false
+ MaxSyncFiles *int `json:"maxSyncFiles,omitempty"`
+
+ // MaxSyncTotalSize is the maximum total size in bytes for all synced files.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ // Default: 52428800 (50MB)
+ //
+ // Required: false
+ MaxSyncTotalSize *int64 `json:"maxSyncTotalSize,omitempty"`
+
+ // MaxSyncBinarySize is the maximum size in bytes for individual binary files.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ // Default: 10485760 (10MB)
+ //
+ // Required: false
+ MaxSyncBinarySize *int64 `json:"maxSyncBinarySize,omitempty"`
}
// UpdateSyncRequest represents the request to update a gitops sync.
@@ -336,6 +395,31 @@ type UpdateSyncRequest struct {
//
// Required: false
SyncInterval *int `json:"syncInterval,omitempty"`
+
+ // SyncDirectory indicates if the entire directory containing the compose file should be synced.
+ // When true, all files in the compose file's directory are synced.
+ // When false, only the compose file itself is synced.
+ //
+ // Required: false
+ SyncDirectory *bool `json:"syncDirectory,omitempty"`
+
+ // MaxSyncFiles is the maximum number of files to sync.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: false
+ MaxSyncFiles *int `json:"maxSyncFiles,omitempty"`
+
+ // MaxSyncTotalSize is the maximum total size in bytes for all synced files.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: false
+ MaxSyncTotalSize *int64 `json:"maxSyncTotalSize,omitempty"`
+
+ // MaxSyncBinarySize is the maximum size in bytes for individual binary files.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: false
+ MaxSyncBinarySize *int64 `json:"maxSyncBinarySize,omitempty"`
}
// SyncResult represents the result of a sync operation.
@@ -581,6 +665,30 @@ type ImportGitOpsSyncRequest struct {
//
// Required: true
SyncInterval int `json:"syncInterval"`
+
+ // SyncDirectory indicates if the entire directory containing the compose file should be synced.
+ // When true (default), all files in the compose file's directory are synced.
+ //
+ // Required: false
+ SyncDirectory *bool `json:"syncDirectory,omitempty"`
+
+ // MaxSyncFiles is the maximum number of files to sync.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: false
+ MaxSyncFiles *int `json:"maxSyncFiles,omitempty"`
+
+ // MaxSyncTotalSize is the maximum total size in bytes for all synced files.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: false
+ MaxSyncTotalSize *int64 `json:"maxSyncTotalSize,omitempty"`
+
+ // MaxSyncBinarySize is the maximum size in bytes for individual binary files.
+ // 0 means inherit the environment limit; unlimited only applies when the effective environment limit is also 0.
+ //
+ // Required: false
+ MaxSyncBinarySize *int64 `json:"maxSyncBinarySize,omitempty"`
}
// ImportGitOpsSyncResponse represents the response for importing gitops syncs.
diff --git a/types/project/project.go b/types/project/project.go
index f690a6491..a7d2ff210 100644
--- a/types/project/project.go
+++ b/types/project/project.go
@@ -238,6 +238,12 @@ type Details struct {
// Required: false
IncludeFiles []IncludeFile `json:"includeFiles,omitempty"`
+ // DirectoryFiles contains all other files in the project directory
+ // (excluding compose files, .env, and include files which are shown separately).
+ //
+ // Required: false
+ DirectoryFiles []IncludeFile `json:"directoryFiles,omitempty"`
+
// Status is the current status of the project.
//
// Required: true
diff --git a/types/settings/settings.go b/types/settings/settings.go
index a924836f9..7a2097c02 100644
--- a/types/settings/settings.go
+++ b/types/settings/settings.go
@@ -137,6 +137,24 @@ type Update struct {
// Required: false
MaxImageUploadSize *string `json:"maxImageUploadSize,omitempty"`
+ // GitSyncMaxFiles is the maximum number of repository files copied during a Git sync.
+ // Set to "0" to disable the environment cap.
+ //
+ // Required: false
+ GitSyncMaxFiles *string `json:"gitSyncMaxFiles,omitempty"`
+
+ // GitSyncMaxTotalSizeMb is the maximum combined size in megabytes for files copied during a Git sync.
+ // Set to "0" to disable the environment cap.
+ //
+ // Required: false
+ GitSyncMaxTotalSizeMb *string `json:"gitSyncMaxTotalSizeMb,omitempty"`
+
+ // GitSyncMaxBinarySizeMb is the maximum size in megabytes for a single binary file copied during a Git sync.
+ // Set to "0" to disable the environment cap.
+ //
+ // Required: false
+ GitSyncMaxBinarySizeMb *string `json:"gitSyncMaxBinarySizeMb,omitempty"`
+
// BaseServerURL is the base URL of the server.
//
// Required: false