Skip to content

Modern Docker Management, Designed for Everyone

fix: recreate namespace dependents when updating Compose services (#4179)

Kyle Mendell33216715407a
diff --git a/.custom-gcl.yml b/.custom-gcl.yml
index 7f4941e9b..39aae7321 100644
--- a/.custom-gcl.yml
+++ b/.custom-gcl.yml
@@ -1,4 +1,4 @@
-version: v2.13.0
+version: v2.13.2
name: golangci-lint-custom
destination: ./.bin
diff --git a/backend/internal/project/project.go b/backend/internal/project/project.go
index 02d425b7e..be6eb3832 100644
--- a/backend/internal/project/project.go
+++ b/backend/internal/project/project.go
@@ -313,7 +313,7 @@ type registryCredentialsProviderInternal func(context.Context) ([]containerregis
func NewProjectService(db *database.DB, settingsService *settings.SettingsService, eventService *event.EventService, imageService *image.ImageService, dockerService *docker.DockerClientService, buildService buildServiceInternal, lifecycleService *LifecycleService, containerRegistryService *registry.ContainerRegistryService, cfg *config.Config) *ProjectService {
return &ProjectService{
- composeCoordinator: projects.NewCoordinator(projecttypes.ComposeCommands{Stop: composeStopProjectServicesInternal, Up: composeUpProjectServicesInternal}),
+ composeCoordinator: projects.NewCoordinator(projecttypes.ComposeCommands{Stop: composeStopProjectServicesInternal, Up: composeUpProjectServicesInternal, Create: projects.ComposeCreate}),
db: db,
settingsService: settingsService,
eventService: eventService,
diff --git a/backend/internal/project/project_lifecycle.go b/backend/internal/project/project_lifecycle.go
index 8cda5ba01..198437842 100644
--- a/backend/internal/project/project_lifecycle.go
+++ b/backend/internal/project/project_lifecycle.go
@@ -107,10 +107,18 @@ func (s *ProjectService) updateProjectServicesInternal(ctx context.Context, proj
previousStatus := projectFromDb.Status
// 1. Load project
- compProj, _, err := s.loadComposeProjectForProjectInternal(ctx, projectFromDb, prepareProjectBindDirectoriesInternal(projectFromDb.Path), servicesToUpdate...)
+ prepare := prepareProjectBindDirectoriesInternal(projectFromDb.Path)
+ compProj, _, err := s.loadComposeProjectForProjectInternal(ctx, projectFromDb, prepare, servicesToUpdate...)
if err != nil {
return errors.WrapIf(err, "failed to load compose project")
}
+ dependents, stoppedDependents := s.existingNamespaceDependentsInternal(ctx, compProj, servicesToUpdate)
+ if len(dependents)+len(stoppedDependents) > 0 {
+ slog.InfoContext(ctx, "recreating namespace dependents with updated services", "projectID", projectID, "services", servicesToUpdate, "dependents", dependents, "stoppedDependents", stoppedDependents)
+ if compProj, _, err = s.loadComposeProjectForProjectInternal(ctx, projectFromDb, prepare, slices.Concat(servicesToUpdate, dependents, stoppedDependents)...); err != nil {
+ return errors.WrapIf(err, "failed to load compose project with dependents")
+ }
+ }
defer s.eventService.BeginComposeSuppressionWindow(compProj.Name)()
@@ -129,7 +137,7 @@ func (s *ProjectService) updateProjectServicesInternal(ctx context.Context, proj
progressWriter, _ := ctx.Value(dockerutil.ProgressWriterKey{}).(io.Writer)
if err := s.composeCoordinator.UpdateServices(ctx, projecttypes.ComposeServiceUpdate{
- Project: compProj, Services: servicesToUpdate,
+ Project: compProj, Services: servicesToUpdate, Dependents: dependents, StoppedDependents: stoppedDependents,
Images: s.composeImageOperationsInternal(&user, credentials), Progress: progressWriter,
AuthConfigs: s.composeRegistryAuthConfigsInternal(ctx), WaitTimeout: s.deployWaitTimeoutInternal(),
RestoreBeforeMutation: func(ctx context.Context) {
@@ -153,11 +161,53 @@ func (s *ProjectService) updateProjectServicesInternal(ctx context.Context, proj
"projectName": projectFromDb.Name,
"services": append([]string(nil), servicesToUpdate...),
}
+ if len(dependents) > 0 {
+ metadata["dependents"] = dependents
+ }
+ if len(stoppedDependents) > 0 {
+ metadata["stoppedDependents"] = stoppedDependents
+ }
s.logProjectEventInternal(ctx, event.EventTypeProjectUpdate, projectID, projectFromDb.Name, user, metadata, "could not log project service update action")
return nil
}
+// existingNamespaceDependentsInternal splits projects.NamespaceDependents into
+// services with a running container and services with only stopped ones, so a
+// stopped dependent is recreated but not started, and a declared-but-never-
+// created dependent is left alone. A ComposePs failure treats every dependent
+// as running: a dangling namespace reference is the worse outcome.
+func (s *ProjectService) existingNamespaceDependentsInternal(ctx context.Context, compProj *composetypes.Project, services []string) (running, stopped []string) {
+ if len(services) == 0 {
+ return nil, nil
+ }
+ deps := projects.NamespaceDependents(compProj, services)
+ if len(deps) == 0 {
+ return nil, nil
+ }
+ summaries, err := projects.ComposePs(ctx, s.dockerService.DockerHost(), compProj, deps, true)
+ if err != nil {
+ slog.WarnContext(ctx, "failed to list namespace dependent containers; recreating all dependents", "project", compProj.Name, "dependents", deps, "error", err)
+ return deps, nil
+ }
+ for _, name := range deps {
+ var found, isRunning bool
+ for _, summary := range summaries {
+ if summary.Service == name {
+ found = true
+ isRunning = isRunning || summary.State == "running"
+ }
+ }
+ switch {
+ case isRunning:
+ running = append(running, name)
+ case found:
+ stopped = append(stopped, name)
+ }
+ }
+ return running, stopped
+}
+
// prepareProjectBindDirectoriesInternal returns the load-time preparation for
// deployments: missing bind-mount sources inside projectPath are created as
// Arcane's runtime user before any container is stopped or created. Left to
diff --git a/backend/internal/project/service_test.go b/backend/internal/project/service_test.go
index 980e9310c..0b2121179 100644
--- a/backend/internal/project/service_test.go
+++ b/backend/internal/project/service_test.go
@@ -1113,7 +1113,9 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T
RepoTags: []string{imageRef},
RepoDigests: []string{repository + "@" + imageDigest},
},
- }, nil)
+ }, func(fullRef string, _ string) {
+ assert.Equal(t, imageRef, fullRef, "namespace dependents must not be pulled")
+ })
dockerService := &docker.DockerClientService{Client: newTestDockerClientInternal(t, server)}
imageUpdateService := imageupdate.NewImageUpdateService(db, nil, nil, dockerService, nil, nil, nil)
@@ -1121,7 +1123,22 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T
imageService := image.NewImageService(db, dockerService, nil, imageUpdateService, nil, eventService)
projectPath := createComposeProjectDir(t, projectsDir, "compose-update-force")
- require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services:\n app:\n image: "+imageRef+"\n labels:\n com.getarcaneapp.arcane.updater.strategy: digest\n unrelated:\n image: busybox:latest\n"), 0o644))
+ require.NoError(t, os.WriteFile(filepath.Join(projectPath, "compose.yaml"), []byte("services:\n app:\n image: "+imageRef+"\n labels:\n com.getarcaneapp.arcane.updater.strategy: digest\n sidecar:\n image: busybox:latest\n network_mode: service:app\n dormant:\n image: busybox:latest\n network_mode: service:app\n never:\n image: busybox:latest\n pid: service:app\n gated:\n image: busybox:latest\n network_mode: service:app\n profiles: [extra]\n unrelated:\n image: busybox:latest\n"), 0o644))
+
+ composeContainerInternal := func(service string, state container.ContainerState) container.Summary {
+ return container.Summary{
+ ID: service + "-container",
+ Names: []string{"/compose-update-force-" + service + "-1"},
+ State: state,
+ Labels: map[string]string{composeapi.ProjectLabel: "compose-update-force", composeapi.ServiceLabel: service, composeapi.ConfigHashLabel: service + "-hash"},
+ }
+ }
+ runtimeServer := newProjectRuntimeDockerServerInternal(t, []container.Summary{
+ composeContainerInternal("app", container.StateRunning),
+ composeContainerInternal("sidecar", container.StateRunning),
+ composeContainerInternal("dormant", container.StateExited),
+ })
+ t.Setenv("DOCKER_HOST", dockerHostFromProjectRuntimeServerURLInternal(t, runtimeServer.URL))
projectRecord := &Project{
ID: "project-update-force",
@@ -1138,7 +1155,9 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T
composeStopProjectServicesInternal = originalComposeStop
composeUpProjectServicesInternal = originalComposeUp
})
- composeStopProjectServicesInternal = func(context.Context, *composetypes.Project, []string) error {
+ var stopped []string
+ composeStopProjectServicesInternal = func(_ context.Context, _ *composetypes.Project, services []string) error {
+ stopped = services
return nil
}
upCalled := false
@@ -1147,9 +1166,9 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T
assert.True(t, eventService.ShouldSuppressDaemonEvent("container", "replacement", "app", selected.Name))
assert.False(t, eventService.ShouldSuppressDaemonEvent("image", "pulled-image", "", ""))
upCalled = true
- assert.Equal(t, []string{"app"}, selected.ServiceNames())
+ assert.Equal(t, []string{"app", "dormant", "sidecar"}, selected.ServiceNames(), "stopped dependents stay in the model for recreation")
forceRecreate = force
- assert.Equal(t, []string{"app"}, services)
+ assert.Equal(t, []string{"app", "sidecar"}, services, "stopped dependents are not started")
assert.False(t, removeOrphans)
return errors.New("compose up failed after assertion")
}
@@ -1160,6 +1179,7 @@ func TestProjectService_UpdateProjectServicesForcesRecreateInternal(t *testing.T
assert.True(t, eventService.ShouldSuppressDaemonEvent("container", "replacement", "app", "compose-update-force"), "failed updates retain correlation through rollback grace")
assert.False(t, eventService.ShouldSuppressDaemonEvent("container", "unrelated", "unrelated", "other-project"))
assert.True(t, upCalled)
+ assert.Equal(t, []string{"app", "sidecar"}, stopped)
assert.True(t, forceRecreate, "service updates must force recreate after pulling the updated image")
}
diff --git a/backend/pkg/projects/cmds.go b/backend/pkg/projects/cmds.go
index 100982d1d..bb86e9a20 100644
--- a/backend/pkg/projects/cmds.go
+++ b/backend/pkg/projects/cmds.go
@@ -114,6 +114,26 @@ func ComposeUp(ctx context.Context, proj *composetypes.Project, services []strin
return c.svc.Up(composeCtx, proj, api.UpOptions{Create: upOptions, Start: startOptions})
}
+// ComposeCreate force-recreates services without starting them, so a stopped
+// container picks up config changes (a recreated namespace provider, for one)
+// and stays stopped.
+func ComposeCreate(ctx context.Context, proj *composetypes.Project, services []string, authConfigs map[string]registry.AuthConfig) error {
+ selected, err := SelectServices(proj, services)
+ if err != nil {
+ return err
+ }
+ createCtx, cancel := detachFromHTTPContextInternal(ctx, defaultComposeTimeout)
+ defer cancel()
+
+ c, err := NewClient(createCtx, "", authConfigs, nil)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = c.Close() }()
+
+ return c.svc.Create(createCtx, selected, api.CreateOptions{Services: services, Recreate: api.RecreateForce, RecreateDependencies: api.RecreateDiverged})
+}
+
func composeUpOptionsInternal(proj *composetypes.Project, services []string, removeOrphans bool, forceRecreate bool, waitTimeout time.Duration, envOpts ComposeEnvOptions) (api.CreateOptions, api.StartOptions) {
recreatePolicy := api.RecreateDiverged
if forceRecreate {
diff --git a/backend/pkg/projects/coordinator.go b/backend/pkg/projects/coordinator.go
index 1ac9d9fde..30a9d1bcc 100644
--- a/backend/pkg/projects/coordinator.go
+++ b/backend/pkg/projects/coordinator.go
@@ -4,6 +4,8 @@ import (
"context"
"io"
"log/slog"
+ "maps"
+ "slices"
"strings"
"time"
@@ -25,6 +27,9 @@ func NewCoordinator(commands projecttypes.ComposeCommands) projecttypes.ComposeC
if commands.Up == nil {
commands.Up = ComposeUp
}
+ if commands.Create == nil {
+ commands.Create = ComposeCreate
+ }
return &composeCoordinatorInternal{commands: commands}
}
@@ -83,7 +88,8 @@ func ResolveRemoveOrphans(gitOpsManaged bool, options *projecttypes.DeployOption
// UpdateServices validates scope before pulling or stopping any service.
func (c *composeCoordinatorInternal) UpdateServices(ctx context.Context, request projecttypes.ComposeServiceUpdate) error {
- selected, err := SelectServices(request.Project, request.Services)
+ scope := slices.Concat(request.Services, request.Dependents)
+ selected, err := SelectServices(request.Project, slices.Concat(scope, request.StoppedDependents))
if err != nil {
if request.RestoreBeforeMutation != nil {
request.RestoreBeforeMutation(ctx)
@@ -96,15 +102,25 @@ func (c *composeCoordinatorInternal) UpdateServices(ctx context.Context, request
}
return errors.WrapIf(err, "pull updated service images")
}
- if err := c.commands.Stop(ctx, selected, request.Services); err != nil {
+ if err := c.commands.Stop(ctx, selected, scope); err != nil {
slog.WarnContext(ctx, "compose stop failed, continuing", "error", err)
}
- if err := c.commands.Up(ctx, selected, request.Services, false, true, false, request.AuthConfigs, request.WaitTimeout); err != nil {
+ if err := c.commands.Up(ctx, selected, scope, false, true, false, request.AuthConfigs, request.WaitTimeout); err != nil {
if request.Recover != nil {
request.Recover(ctx)
}
return errors.WrapIf(err, "failed to up services")
}
+ // Stopped dependents are recreated after the up so they resolve the new
+ // provider containers, and are left stopped.
+ if len(request.StoppedDependents) > 0 {
+ if err := c.commands.Create(ctx, selected, request.StoppedDependents, request.AuthConfigs); err != nil {
+ if request.Recover != nil {
+ request.Recover(ctx)
+ }
+ return errors.WrapIf(err, "failed to recreate stopped dependents")
+ }
+ }
return nil
}
@@ -124,6 +140,55 @@ func SelectServices(model *composetypes.Project, services []string) (*composetyp
return selected.WithoutUnnecessaryResources(), nil
}
+// NamespaceDependents returns the services outside services that join a member's
+// network/ipc/pid namespace or use volumes_from it, transitively. Mirrors compose's
+// parentNamespaceRecreated cascade; services inactive for model.Profiles are skipped.
+func NamespaceDependents(model *composetypes.Project, services []string) []string {
+ if model == nil {
+ return nil
+ }
+ members := make(map[string]struct{}, len(services))
+ for _, name := range services {
+ members[name] = struct{}{}
+ }
+ added := map[string]struct{}{}
+ all := model.AllServices()
+ joins := func(svc composetypes.ServiceConfig) bool {
+ for _, mode := range []string{svc.NetworkMode, svc.Ipc, svc.Pid} {
+ if name, ok := strings.CutPrefix(mode, composetypes.ServicePrefix); ok {
+ if _, member := members[name]; member {
+ return true
+ }
+ }
+ }
+ for _, vol := range svc.VolumesFrom {
+ if strings.HasPrefix(vol, composetypes.ContainerPrefix) {
+ continue
+ }
+ name, _, _ := strings.Cut(vol, ":")
+ if _, member := members[name]; member {
+ return true
+ }
+ }
+ return false
+ }
+ for changed := true; changed; {
+ changed = false
+ for name, svc := range all {
+ if _, ok := members[name]; ok || !svc.HasProfile(model.Profiles) || !joins(svc) {
+ continue
+ }
+ members[name] = struct{}{}
+ added[name] = struct{}{}
+ changed = true
+ }
+ }
+ if len(added) == 0 {
+ return nil
+ }
+ return slices.Sorted(maps.Keys(added))
+}
+
// PullServices pulls each selected registry image once through the application image service.
func (c *composeCoordinatorInternal) PullServices(ctx context.Context, model *composetypes.Project, services []string, operations projecttypes.ComposeImageOperations, progress io.Writer) error {
for _, imageRef := range SelectedImageRefs(model, services) {
diff --git a/backend/pkg/projects/coordinator_test.go b/backend/pkg/projects/coordinator_test.go
new file mode 100644
index 000000000..18c683333
--- /dev/null
+++ b/backend/pkg/projects/coordinator_test.go
@@ -0,0 +1,140 @@
+package projects
+
+import (
+ "context"
+ "io"
+ "testing"
+ "time"
+
+ composetypes "github.com/compose-spec/compose-go/v2/types"
+ projecttypes "github.com/getarcaneapp/arcane/types/v2/project"
+ "github.com/moby/moby/api/types/registry"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestNamespaceDependents(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ enabled composetypes.Services
+ disabled composetypes.Services
+ profiles []string
+ services []string
+ want []string
+ }{
+ {
+ name: "transitive network_mode chain",
+ enabled: composetypes.Services{"a": {Name: "a"}},
+ disabled: composetypes.Services{"b": {Name: "b", NetworkMode: "service:a"}, "c": {Name: "c", NetworkMode: "service:b"}, "d": {Name: "d"}},
+ services: []string{"a"},
+ want: []string{"b", "c"},
+ },
+ {
+ name: "ipc and pid refs",
+ enabled: composetypes.Services{"a": {Name: "a"}},
+ disabled: composetypes.Services{"z": {Name: "z", Ipc: "service:a"}, "y": {Name: "y", Pid: "service:a"}, "x": {Name: "x", Ipc: "shareable"}},
+ services: []string{"a"},
+ want: []string{"y", "z"},
+ },
+ {
+ name: "volumes_from service matched and container skipped",
+ enabled: composetypes.Services{"a": {Name: "a"}},
+ disabled: composetypes.Services{"b": {Name: "b", VolumesFrom: []string{"a:ro"}}, "c": {Name: "c", VolumesFrom: []string{"container:a"}}},
+ services: []string{"a"},
+ want: []string{"b"},
+ },
+ {
+ name: "profile-gated dependent excluded without profile",
+ enabled: composetypes.Services{"a": {Name: "a"}},
+ disabled: composetypes.Services{"b": {Name: "b", NetworkMode: "service:a", Profiles: []string{"extra"}}},
+ services: []string{"a"},
+ },
+ {
+ name: "profile-gated dependent included with profile",
+ enabled: composetypes.Services{"a": {Name: "a"}},
+ disabled: composetypes.Services{"b": {Name: "b", NetworkMode: "service:a", Profiles: []string{"extra"}}},
+ profiles: []string{"extra"},
+ services: []string{"a"},
+ want: []string{"b"},
+ },
+ {
+ name: "inputs are excluded",
+ enabled: composetypes.Services{"a": {Name: "a"}, "b": {Name: "b", NetworkMode: "service:a"}},
+ disabled: composetypes.Services{"c": {Name: "c", NetworkMode: "service:b"}},
+ services: []string{"a", "b"},
+ want: []string{"c"},
+ },
+ {
+ name: "nil when nothing matches",
+ enabled: composetypes.Services{"a": {Name: "a"}},
+ disabled: composetypes.Services{"b": {Name: "b", NetworkMode: "host"}},
+ services: []string{"a"},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ model := &composetypes.Project{Services: tt.enabled, DisabledServices: tt.disabled, Profiles: tt.profiles}
+ assert.Equal(t, tt.want, NamespaceDependents(model, tt.services))
+ })
+ }
+}
+
+func TestCoordinatorUpdateServicesScopesDependents(t *testing.T) {
+ t.Parallel()
+
+ var stopped, upped, created, pulled, calls []string
+ var selectedNames []string
+ forceRecreate := false
+ coordinator := NewCoordinator(projecttypes.ComposeCommands{
+ Stop: func(_ context.Context, _ *composetypes.Project, services []string) error {
+ stopped = services
+ calls = append(calls, "stop")
+ return nil
+ },
+ Up: func(_ context.Context, selected *composetypes.Project, services []string, _ bool, force bool, _ bool, _ map[string]registry.AuthConfig, _ time.Duration) error {
+ upped = services
+ selectedNames = selected.ServiceNames()
+ forceRecreate = force
+ calls = append(calls, "up")
+ return nil
+ },
+ Create: func(_ context.Context, selected *composetypes.Project, services []string, _ map[string]registry.AuthConfig) error {
+ created = services
+ assert.Equal(t, selectedNames, selected.ServiceNames(), "create reuses the scoped model")
+ calls = append(calls, "create")
+ return nil
+ },
+ })
+
+ model := &composetypes.Project{
+ Name: "stack",
+ Services: composetypes.Services{
+ "app": {Name: "app", Image: "example.com/app:2"},
+ "sidecar": {Name: "sidecar", Image: "example.com/sidecar:1", NetworkMode: "service:app"},
+ "dormant": {Name: "dormant", Image: "example.com/dormant:1", Pid: "service:app"},
+ },
+ }
+ err := coordinator.UpdateServices(context.Background(), projecttypes.ComposeServiceUpdate{
+ Project: model,
+ Services: []string{"app"},
+ Dependents: []string{"sidecar"},
+ StoppedDependents: []string{"dormant"},
+ Images: projecttypes.ComposeImageOperations{Pull: func(_ context.Context, ref string, _ io.Writer) error {
+ pulled = append(pulled, ref)
+ return nil
+ }},
+ })
+ require.NoError(t, err)
+
+ assert.Equal(t, []string{"example.com/app:2"}, pulled)
+ assert.Equal(t, []string{"app", "sidecar"}, stopped)
+ assert.Equal(t, []string{"app", "sidecar"}, upped, "stopped dependents are not started")
+ assert.Equal(t, []string{"app", "dormant", "sidecar"}, selectedNames)
+ assert.True(t, forceRecreate)
+ assert.Equal(t, []string{"dormant"}, created)
+ assert.Equal(t, []string{"stop", "up", "create"}, calls, "stopped dependents are recreated after the providers")
+}
diff --git a/types/project/compose_operation.go b/types/project/compose_operation.go
index f9ce580ad..78bab71e6 100644
--- a/types/project/compose_operation.go
+++ b/types/project/compose_operation.go
@@ -45,8 +45,13 @@ type ComposeDeployment struct {
// ComposeServiceUpdate supplies an already authorized service update.
type ComposeServiceUpdate struct {
- Project *composetypes.Project
- Services []string
+ Project *composetypes.Project
+ Services []string
+ // Dependents are recreated alongside Services because they share a member's
+ // namespace or volumes; they are never pulled. StoppedDependents are
+ // recreated without being started so an operator's stop is preserved.
+ Dependents []string
+ StoppedDependents []string
Images ComposeImageOperations
Progress io.Writer
AuthConfigs map[string]registry.AuthConfig
@@ -57,8 +62,9 @@ type ComposeServiceUpdate struct {
// ComposeCommands supplies the SDK operations used by the coordinator.
type ComposeCommands struct {
- Stop func(context.Context, *composetypes.Project, []string) error
- Up func(context.Context, *composetypes.Project, []string, bool, bool, bool, map[string]registry.AuthConfig, time.Duration) error
+ Stop func(context.Context, *composetypes.Project, []string) error
+ Up func(context.Context, *composetypes.Project, []string, bool, bool, bool, map[string]registry.AuthConfig, time.Duration) error
+ Create func(context.Context, *composetypes.Project, []string, map[string]registry.AuthConfig) error
}
// ComposeCoordinator executes Compose workflows through application collaborators.