Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
feat: implement multi-file swarm git sync and host path mapping (#2457)
Lewis Englandf7355c5d198f
diff --git a/backend/internal/services/gitops_sync_service.go b/backend/internal/services/gitops_sync_service.goindex e2af07270..585f0389d 100644--- a/backend/internal/services/gitops_sync_service.go+++ b/backend/internal/services/gitops_sync_service.go@@ -612,10 +612,30 @@ func (s *GitOpsSyncService) performSwarmStackSyncInternal(ctx context.Context, senvContent = *source.envContent}+ var syncFiles []projects.SyncFile+ if sync.SyncDirectory {+ _, files, err := s.walkAndParseSyncDirectory(ctx, sync, source.repoPath)+ if err != nil {+ return result, s.failSync(ctx, id, result, sync, actor, "Failed to walk directory", err.Error())+ }+ syncFiles = files+ }++ swarmFiles := make([]swarm.SyncFile, len(syncFiles))+ syncedFiles := make([]string, 0, len(syncFiles))+ for i, f := range syncFiles {+ swarmFiles[i] = swarm.SyncFile{+ RelativePath: f.RelativePath,+ Content: f.Content,+ }+ syncedFiles = append(syncedFiles, f.RelativePath)+ }+req := swarm.StackDeployRequest{Name: sync.ProjectName,ComposeContent: source.composeContent,EnvContent: envContent,+ Files: swarmFiles,Prune: true,WorkingDir: filepath.Dir(filepath.Join(source.repoPath, sync.ComposePath)),}@@ -624,7 +644,9 @@ func (s *GitOpsSyncService) performSwarmStackSyncInternal(ctx context.Context, sreturn result, s.failSync(ctx, id, result, sync, actor, "Failed to deploy swarm stack", err.Error())}- syncedFiles := []string{filepath.Base(sync.ComposePath)}+ if len(syncedFiles) == 0 {+ syncedFiles = []string{filepath.Base(sync.ComposePath)}+ }s.updateSyncStatusWithFiles(ctx, id, "success", "", source.commitHash, syncedFiles)result.Success = trueresult.Message = fmt.Sprintf("Successfully deployed swarm stack %s from %s", sync.ProjectName, sync.ComposePath)diff --git a/backend/internal/services/swarm_service.go b/backend/internal/services/swarm_service.goindex 30319eeeb..60273a915 100644--- a/backend/internal/services/swarm_service.go+++ b/backend/internal/services/swarm_service.go@@ -6,6 +6,7 @@ import ("encoding/json""errors""fmt"+ "io/fs""log/slog""net/http""net/netip"@@ -20,6 +21,7 @@ import (cerrdefs "github.com/containerd/errdefs""github.com/getarcaneapp/arcane/backend/internal/common""github.com/getarcaneapp/arcane/backend/internal/models"+ dockerutil "github.com/getarcaneapp/arcane/backend/pkg/dockerutil""github.com/getarcaneapp/arcane/backend/pkg/libarcane/edge"libswarm "github.com/getarcaneapp/arcane/backend/pkg/libarcane/swarm""github.com/getarcaneapp/arcane/backend/pkg/pagination"@@ -836,6 +838,25 @@ func (s *SwarmService) DeployStack(ctx context.Context, environmentID string, rereturn nil, fmt.Errorf("failed to connect to Docker: %w", err)}+ _, stackSourceDir, err := s.resolveSwarmStackSourceDirInternal(ctx, environmentID, stackName)+ if err != nil {+ return nil, err+ }++ if err := s.upsertStackSourceInternal(ctx, environmentID, stackName, req.ComposeContent, req.EnvContent, req.Files); err != nil {+ slog.WarnContext(ctx, "failed to persist swarm stack source", "environmentID", normalizeSwarmEnvironmentIDInternal(environmentID), "stackName", stackName, "error", err)+ }++ workingDir := req.WorkingDir+ if workingDir == "" || len(req.Files) > 0 {+ workingDir = stackSourceDir+ }++ pm, err := s.getPathMapperInternal(ctx)+ if err != nil {+ slog.WarnContext(ctx, "failed to initialize path mapper, deploying without path translation", "error", err)+ }+if err := libswarm.DeployStack(ctx, dockerClient, libswarm.StackDeployOptions{Name: stackName,ComposeContent: req.ComposeContent,@@ -849,15 +870,12 @@ func (s *SwarmService) DeployStack(ctx context.Context, environmentID string, re},Prune: req.Prune,ResolveImage: req.ResolveImage,- WorkingDir: req.WorkingDir,+ WorkingDir: workingDir,+ PathMapper: pm,}); err != nil {return nil, err}- if err := s.upsertStackSourceInternal(ctx, environmentID, stackName, req.ComposeContent, req.EnvContent); err != nil {- slog.WarnContext(ctx, "failed to persist swarm stack source", "environmentID", normalizeSwarmEnvironmentIDInternal(environmentID), "stackName", stackName, "error", err)- }-return &swarmtypes.StackDeployResponse{Name: stackName}, nil}@@ -1327,10 +1345,43 @@ func (s *SwarmService) GetStackSource(ctx context.Context, environmentID, stackNreturn nil, fmt.Errorf("failed to read swarm stack env source: %w", err)}+ var files []swarmtypes.SyncFile+ root, err := os.OpenRoot(stackSourceDir)+ if err == nil {+ defer func() { _ = root.Close() }()+ err = fs.WalkDir(root.FS(), ".", func(path string, d fs.DirEntry, walkErr error) error {+ if walkErr != nil {+ return walkErr+ }+ if path == "." || d.IsDir() {+ return nil+ }+ if d.Type()&os.ModeSymlink != 0 {+ return nil+ }+ if path == swarmStackComposeFilename || path == swarmStackEnvFilename {+ return nil+ }+ content, err := root.ReadFile(path)+ if err != nil {+ return err+ }+ files = append(files, swarmtypes.SyncFile{+ RelativePath: path,+ Content: content,+ })+ return nil+ })+ }+ if err != nil && !errors.Is(err, os.ErrNotExist) {+ return nil, fmt.Errorf("failed to read additional swarm stack source files: %w", err)+ }+return &swarmtypes.StackSource{Name: stackName,ComposeContent: string(composeContent),EnvContent: envContent,+ Files: files,}, nil}@@ -1343,7 +1394,7 @@ func (s *SwarmService) UpdateStackSource(ctx context.Context, environmentID, stareturn nil, errors.New("stack compose source is required")}- if err := s.upsertStackSourceInternal(ctx, environmentID, stackName, req.ComposeContent, req.EnvContent); err != nil {+ if err := s.upsertStackSourceInternal(ctx, environmentID, stackName, req.ComposeContent, req.EnvContent, req.Files); err != nil {return nil, err}@@ -1351,6 +1402,7 @@ func (s *SwarmService) UpdateStackSource(ctx context.Context, environmentID, staName: stackName,ComposeContent: req.ComposeContent,EnvContent: req.EnvContent,+ Files: req.Files,}, nil}@@ -1611,10 +1663,16 @@ func (s *SwarmService) RenderStackConfig(ctx context.Context, req swarmtypes.Stareturn nil, err}+ pm, err := s.getPathMapperInternal(ctx)+ if err != nil {+ slog.WarnContext(ctx, "failed to initialize path mapper, deploying without path translation", "error", err)+ }+result, err := libswarm.RenderStackConfig(ctx, libswarm.StackRenderOptions{Name: req.Name,ComposeContent: req.ComposeContent,EnvContent: req.EnvContent,+ PathMapper: pm,})if err != nil {return nil, err@@ -2055,7 +2113,7 @@ func decodeSecretSpecInternal(raw json.RawMessage) (swarm.SecretSpec, error) {return spec, nil}-func (s *SwarmService) upsertStackSourceInternal(ctx context.Context, environmentID, stackName, composeContent, envContent string) error {+func (s *SwarmService) upsertStackSourceInternal(ctx context.Context, environmentID, stackName, composeContent, envContent string, files []swarmtypes.SyncFile) error {stackName = strings.TrimSpace(stackName)if stackName == "" {return errors.New("stack name is required")@@ -2075,15 +2133,32 @@ func (s *SwarmService) upsertStackSourceInternal(ctx context.Context, environmen}envPath := filepath.Join(stackSourceDir, swarmStackEnvFilename)- if envContent == "" {+ if envContent != "" {+ if err := os.WriteFile(envPath, []byte(envContent), common.FilePerm); err != nil {+ return fmt.Errorf("failed to write swarm stack env source: %w", err)+ }+ } else {if err := os.Remove(envPath); err != nil && !errors.Is(err, os.ErrNotExist) {return fmt.Errorf("failed to clear swarm stack env source: %w", err)}- return nil}- if err := os.WriteFile(envPath, []byte(envContent), common.FilePerm); err != nil {- return fmt.Errorf("failed to write swarm stack env source: %w", err)+ // Write additional files+ for _, f := range files {+ if f.RelativePath == swarmStackComposeFilename || f.RelativePath == swarmStackEnvFilename {+ continue+ }+ fPath := filepath.Join(stackSourceDir, f.RelativePath)+ // Prevent path traversal+ if !appfs.IsSafeSubdirectory(stackSourceDir, fPath) {+ return fmt.Errorf("invalid file path %q: must be inside stack directory", f.RelativePath)+ }+ if err := os.MkdirAll(filepath.Dir(fPath), common.DirPerm); err != nil {+ return fmt.Errorf("failed to create directory for %s: %w", f.RelativePath, err)+ }+ if err := os.WriteFile(fPath, f.Content, common.FilePerm); err != nil {+ return fmt.Errorf("failed to write %s: %w", f.RelativePath, err)+ }}return nil@@ -2177,6 +2252,55 @@ func (s *SwarmService) resolveSwarmStackSourceEnvironmentDirInternal(ctx contextreturn rootDir, environmentDir, nil}+func (s *SwarmService) getPathMapperInternal(ctx context.Context) (*appfs.PathMapper, error) {+ configuredPath := s.settingsService.GetStringSetting(ctx, "swarmStackSourcesDirectory", defaultSwarmStackSourceRootDir)++ var containerDir, hostDir string++ // Handle mapping format: "container_path:host_path"+ if parts := strings.SplitN(configuredPath, ":", 2); len(parts) == 2 {+ if !appfs.IsWindowsDrivePath(configuredPath) && strings.HasPrefix(parts[0], "/") {+ containerDir = parts[0]+ hostDir = parts[1]+ }+ }++ if containerDir == "" {+ containerDir = configuredPath+ }++ // Resolve container directory to absolute path+ containerDirResolved, err := appfs.GetProjectsDirectory(ctx, strings.TrimSpace(containerDir))+ if err != nil {+ slog.WarnContext(ctx, "unable to resolve container swarm sources directory, using default", "error", err)+ containerDirResolved = defaultSwarmStackSourceRootDir+ }++ // If hostDir not obtained from mapping, attempt auto-discovery from Docker mounts+ if hostDir == "" && s.dockerService != nil {+ if dockerCli, derr := s.dockerService.GetClient(ctx); derr == nil {+ absContainerDir, _ := filepath.Abs(containerDirResolved)+ if discovery, aerr := dockerutil.GetHostPathForContainerPath(ctx, dockerCli, absContainerDir); aerr == nil && discovery != "" {+ hostDir = discovery+ slog.DebugContext(ctx, "Auto-discovered host path for swarm stack sources", "container", absContainerDir, "host", hostDir)+ }+ }+ }++ // Clean host directory+ hostDirResolved := strings.TrimSpace(hostDir)+ if hostDirResolved != "" {+ hostDirResolved = filepath.Clean(hostDirResolved)+ }++ pm := appfs.NewPathMapper(containerDirResolved, hostDirResolved)+ if !pm.IsNonMatchingMount() {+ return nil, nil+ }++ return pm, nil+}+func (s *SwarmService) ensureSwarmManagerInternal(ctx context.Context) error {info, err := s.getDockerInfoInternal(ctx)if err != nil {diff --git a/backend/internal/services/swarm_service_test.go b/backend/internal/services/swarm_service_test.goindex 6d669e4ad..ab01bdd4f 100644--- a/backend/internal/services/swarm_service_test.go+++ b/backend/internal/services/swarm_service_test.go@@ -102,6 +102,56 @@ func TestSwarmService_UpdateAndGetStackSource_UsesStoredFilesWithoutSwarmManagerrequire.NoError(t, err)require.Equal(t, updated.ComposeContent, source.ComposeContent)require.Equal(t, updated.EnvContent, source.EnvContent)++ // Test with additional files+ updated, err = svc.UpdateStackSource(ctx, "0", "demo-stack", swarmtypes.StackSourceUpdateRequest{+ ComposeContent: "services:\n web:\n image: nginx:alpine\n",+ Files: []swarmtypes.SyncFile{+ {RelativePath: "config/nginx.conf", Content: []byte("worker_processes 1;")},+ {RelativePath: "scripts/setup.sh", Content: []byte("#!/bin/sh")},+ },+ })+ require.NoError(t, err)+ require.FileExists(t, filepath.Join(rootDir, "0", "demo-stack", "config", "nginx.conf"))+ require.FileExists(t, filepath.Join(rootDir, "0", "demo-stack", "scripts", "setup.sh"))++ source, err = svc.GetStackSource(ctx, "0", "demo-stack")+ require.NoError(t, err)+ require.Len(t, source.Files, 2)+}++func TestSwarmService_getPathMapperInternal(t *testing.T) {+ ctx := context.Background()+ db := setupSettingsTestDB(t)+ settingsSvc, err := NewSettingsService(ctx, db)+ require.NoError(t, err)++ svc := NewSwarmService(nil, settingsSvc, nil, nil, nil)++ t.Run("returns nil when paths match", func(t *testing.T) {+ pm, err := svc.getPathMapperInternal(ctx)+ require.NoError(t, err)+ require.Nil(t, pm) // Default is /app/data/swarm/sources which matches itself+ })++ t.Run("returns mapper when mapping configured", func(t *testing.T) {+ containerDir := filepath.Join(t.TempDir(), "container")+ hostDir := "/host/path"+ err := settingsSvc.UpdateSetting(ctx, "swarmStackSourcesDirectory", containerDir+":"+hostDir)+ require.NoError(t, err)++ pm, err := svc.getPathMapperInternal(ctx)+ require.NoError(t, err)+ require.NotNil(t, pm)+ require.True(t, pm.IsNonMatchingMount())++ testFile := filepath.Join(containerDir, "0/stack/compose.yaml")+ expected := filepath.Join(hostDir, "0/stack/compose.yaml")++ translated, err := pm.ContainerToHost(testFile)+ require.NoError(t, err)+ require.Equal(t, filepath.ToSlash(expected), filepath.ToSlash(translated))+ })}func TestSwarmService_ScaleService_HandlesServiceModesInternal(t *testing.T) {diff --git a/backend/pkg/dockerutil/mount_utils.go b/backend/pkg/dockerutil/mount_utils.goindex 376c2329f..27d3f82ed 100644--- a/backend/pkg/dockerutil/mount_utils.go+++ b/backend/pkg/dockerutil/mount_utils.go@@ -45,7 +45,7 @@ func GetHostPathForContainerPath(ctx context.Context, dockerCli *client.Client,}}- if bestMatch != nil && bestMatch.Type == mounttypes.TypeBind {+ if bestMatch != nil && (bestMatch.Type == mounttypes.TypeBind || bestMatch.Type == mounttypes.TypeVolume) {// Calculate the relative path from mount destination to target pathrel := strings.TrimPrefix(containerPath, bestMatch.Destination)rel = strings.TrimPrefix(rel, "/") // Ensure no double slashdiff --git a/backend/pkg/libarcane/swarm/stack_deploy_engine.go b/backend/pkg/libarcane/swarm/stack_deploy_engine.goindex e7185c0e3..cf846acf1 100644--- a/backend/pkg/libarcane/swarm/stack_deploy_engine.go+++ b/backend/pkg/libarcane/swarm/stack_deploy_engine.go@@ -53,6 +53,7 @@ type StackDeployOptions struct {Prune boolResolveImage stringWorkingDir string+ PathMapper *projects.PathMapper}type StackRenderOptions struct {@@ -60,6 +61,7 @@ type StackRenderOptions struct {ComposeContent stringEnvContent stringWorkingDir string+ PathMapper *projects.PathMapper}type StackRenderResult struct {@@ -98,7 +100,7 @@ func DeployStack(ctx context.Context, dockerClient *dockerclient.Client, opts Streturn err}- project, err := loadComposeProject(ctx, stackName, opts.ComposeContent, opts.EnvContent, opts.WorkingDir)+ project, err := loadComposeProject(ctx, stackName, opts.ComposeContent, opts.EnvContent, opts.WorkingDir, opts.PathMapper)if err != nil {return err}@@ -252,7 +254,7 @@ func RenderStackConfig(ctx context.Context, opts StackRenderOptions) (*StackRendreturn nil, errors.New("stack name is required")}- project, err := loadComposeProject(ctx, stackName, opts.ComposeContent, opts.EnvContent, opts.WorkingDir)+ project, err := loadComposeProject(ctx, stackName, opts.ComposeContent, opts.EnvContent, opts.WorkingDir, opts.PathMapper)if err != nil {return nil, err}@@ -276,7 +278,7 @@ func RenderStackConfig(ctx context.Context, opts StackRenderOptions) (*StackRend}, nil}-func loadComposeProject(ctx context.Context, projectName, composeContent, envContent, providedWorkingDir string) (*composegotypes.Project, error) {+func loadComposeProject(ctx context.Context, projectName, composeContent, envContent, providedWorkingDir string, pathMapper *projects.PathMapper) (*composegotypes.Project, error) {composeContent = strings.TrimSpace(composeContent)if composeContent == "" {return nil, errors.New("compose content is required")@@ -317,6 +319,17 @@ func loadComposeProject(ctx context.Context, projectName, composeContent, envCon}project = project.WithoutUnnecessaryResources()++ // Resolve relative paths for bind mounts, secrets, and configs+ projects.ResolveRelativeProjectPaths(project, workingDir)++ // Translate container paths to host paths for Docker execution+ if pathMapper != nil {+ if err := pathMapper.TranslateVolumeSources(project); err != nil {+ return nil, fmt.Errorf("failed to translate paths for docker host: %w", err)+ }+ }+return project, nil}diff --git a/backend/pkg/projects/load.go b/backend/pkg/projects/load.goindex 6bba2576c..9130fbfb1 100644--- a/backend/pkg/projects/load.go+++ b/backend/pkg/projects/load.go@@ -239,7 +239,7 @@ func loadComposeProjectInternal(project = project.WithoutUnnecessaryResources()// Resolve relative paths for bind mounts, secrets, and configs- resolveRelativeProjectPaths(project, workdir)+ ResolveRelativeProjectPaths(project, workdir)// Translate container paths to host paths for Docker executionif pathMapper != nil {@@ -296,7 +296,7 @@ func LoadComposeProjectFromDir(ctx context.Context, dir, projectName, projectsDireturn proj, composeFile, nil}-func resolveRelativeProjectPaths(project *composetypes.Project, workdir string) {+func ResolveRelativeProjectPaths(project *composetypes.Project, workdir string) {if project == nil || workdir == "" {return}diff --git a/backend/pkg/projects/load_test.go b/backend/pkg/projects/load_test.goindex 307aefd8d..b3db59e0a 100644--- a/backend/pkg/projects/load_test.go+++ b/backend/pkg/projects/load_test.go@@ -220,3 +220,25 @@ services:require.Equal(t, project.WorkingDir, includedService.CustomLabels[api.WorkingDirLabel])require.Equal(t, expectedConfigFiles, includedService.CustomLabels[api.ConfigFilesLabel])}++func TestResolveRelativeProjectPaths(t *testing.T) {+ t.Parallel()++ dir := t.TempDir()+ composePath := filepath.Join(dir, "compose.yaml")+ require.NoError(t, os.WriteFile(composePath, []byte(`services:+ app:+ image: nginx:alpine+ volumes:+ - ./config.conf:/etc/app/config.conf+`), 0o600))++ project, err := LoadComposeProject(context.Background(), composePath, "demo", dir, false, nil)+ require.NoError(t, err)++ ResolveRelativeProjectPaths(project, dir)++ service := project.Services["app"]+ require.Len(t, service.Volumes, 1)+ assert.Equal(t, filepath.Join(dir, "config.conf"), service.Volumes[0].Source)+}diff --git a/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte b/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelteindex ced5bb582..d269576b6 100644--- a/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte+++ b/frontend/src/lib/components/dialogs/gitops-sync-dialog.svelte@@ -81,7 +81,8 @@name: open && syncToEdit ? syncToEdit.name : '',repositoryId: open && syncToEdit ? syncToEdit.repositoryId : '',branch: open && syncToEdit ? syncToEdit.branch : 'main',- composePath: open && syncToEdit ? syncToEdit.composePath : 'docker-compose.yml',+ composePath:+ open && syncToEdit ? syncToEdit.composePath : selectedTargetType === 'swarm_stack' ? 'compose.yml' : 'docker-compose.yml',syncDirectory: open && syncToEdit ? (syncToEdit.syncDirectory ?? false) : false,maxSyncFiles: open && syncToEdit ? (syncToEdit.maxSyncFiles ?? 0) : (settingsQuery.data?.gitSyncMaxFiles ?? 0),maxSyncTotalSizeMb:@@ -274,7 +275,11 @@<Label for="composePath">{m.git_sync_compose_path()}</Label><div class="flex gap-2"><div class="flex-1">- <FormInput type="text" placeholder="docker-compose.yml" bind:input={$inputs.composePath} />+ <FormInput+ type="text"+ placeholder={selectedTargetType === 'swarm_stack' ? 'compose.yml' : 'docker-compose.yml'}+ bind:input={$inputs.composePath}+ /></div><Buttontype="button"diff --git a/types/swarm/stack.go b/types/swarm/stack.goindex d92d8abb8..7e2370ec5 100644--- a/types/swarm/stack.go+++ b/types/swarm/stack.go@@ -135,6 +135,11 @@ type StackSource struct {//// Required: falseEnvContent string `json:"envContent,omitempty"`++ // Files is an optional list of additional files synced with the stack.+ //+ // Required: false+ Files []SyncFile `json:"files,omitempty"`}type StackSourceUpdateRequest struct {@@ -147,4 +152,9 @@ type StackSourceUpdateRequest struct {//// Required: falseEnvContent string `json:"envContent,omitempty"`++ // Files is an optional list of additional files to persist for the stack.+ //+ // Required: false+ Files []SyncFile `json:"files,omitempty"`}diff --git a/types/swarm/stack_deploy.go b/types/swarm/stack_deploy.goindex 6b8c37bfe..b7cb8e9da 100644--- a/types/swarm/stack_deploy.go+++ b/types/swarm/stack_deploy.go@@ -17,6 +17,11 @@ type StackDeployRequest struct {// Required: falseEnvContent string `json:"envContent,omitempty"`+ // Files is an optional list of additional files to sync (e.g. env_file, configs, secrets).+ //+ // Required: false+ Files []SyncFile `json:"files,omitempty"`+// WithRegistryAuth sends registry auth details to Swarm agents.//// Required: false@@ -38,6 +43,19 @@ type StackDeployRequest struct {WorkingDir string `json:"workingDir,omitempty"`}+// SyncFile represents a file to be synced to the target environment.+type SyncFile struct {+ // RelativePath is the path of the file relative to the stack's working directory.+ //+ // Required: true+ RelativePath string `json:"relativePath"`++ // Content is the file content.+ //+ // Required: true+ Content []byte `json:"content"`+}+// StackDeployResponse represents the result of a stack deployment.type StackDeployResponse struct {// Name is the deployed stack name.