Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: update containers whose image tag was moved by an interrupted run
Kyle Mendell8928c098a978
diff --git a/backend/go.mod b/backend/go.modindex 6d7ddf3fe..dbd0bd9b8 100644--- a/backend/go.mod+++ b/backend/go.mod@@ -73,7 +73,7 @@ require (go.getarcane.app/streams v0.4.6go.getarcane.app/sys/cgroup v0.2.5go.getarcane.app/sys/crypto v0.3.1- go.getarcane.app/updater v0.11.8+ go.getarcane.app/updater v0.11.9go.opentelemetry.io/contrib/bridges/otelslog v0.21.0go.opentelemetry.io/contrib/exporters/autoexport v0.72.0go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0diff --git a/backend/go.sum b/backend/go.sumindex b406ad4bd..0c300a23f 100644--- a/backend/go.sum+++ b/backend/go.sum@@ -771,8 +771,8 @@ go.getarcane.app/sys/cgroup v0.2.5 h1:iI1qURe+kOgTRN75sxkpPePjJwlxAx3c9e9kEwKAVngo.getarcane.app/sys/cgroup v0.2.5/go.mod h1:hJyCjpx04VcaaSvkRsMvZl5OLUlAQ60K5gEy1ctuU4U=go.getarcane.app/sys/crypto v0.3.1 h1:NzTk0IyeL8FKK71m+bL4WpMaSt+uLMZ7N+IWNFbpLnI=go.getarcane.app/sys/crypto v0.3.1/go.mod h1:Rxnq1pb5HVSoJ4VKmL6LZGCrztMONaypXKc8Zg0BMhE=-go.getarcane.app/updater v0.11.8 h1:skyIaDoUPXll10jdrZDoWDl2la/nBHZ1MECXtgiJ5xE=-go.getarcane.app/updater v0.11.8/go.mod h1:kJy2PIstnWGZgvcWcwJeTTwYu6TvZl3APeLhvIoFR5I=+go.getarcane.app/updater v0.11.9 h1:bi7SxRpq/QnP9OE5ZgJOHmoCOoSDi2sssXgMoo5Cqtc=+go.getarcane.app/updater v0.11.9/go.mod h1:kJy2PIstnWGZgvcWcwJeTTwYu6TvZl3APeLhvIoFR5I=go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=diff --git a/backend/internal/updater/children/recovery/service.go b/backend/internal/updater/children/recovery/service.goindex 3492a1d4e..6870953a7 100644--- a/backend/internal/updater/children/recovery/service.go+++ b/backend/internal/updater/children/recovery/service.go@@ -18,6 +18,7 @@ import ("go.getarcane.app/updater/refs""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref")// errNoFrozenPlan reports a run without a persisted frozen plan.@@ -116,7 +117,7 @@ func (s *Service) FreezePending(ctx context.Context, run scheduler.Run, excludedreturn ctx, err}records = slices.DeleteFunc(records, func(record updater.ImageUpdateRecord) bool {- return excluded[refs.NormalizeImageUpdateRef(record.ImageRef())] || excluded[record.ContainerID]+ return !record.HasUpdate || excluded[refs.NormalizeImageUpdateRef(record.ImageRef())] || excluded[record.ContainerID]})failures := map[string]error{}selected := make([]updater.ImageUpdateRecord, 0, len(records))@@ -130,15 +131,17 @@ func (s *Service) FreezePending(ctx context.Context, run scheduler.Run, excludedif listErr != nil {return ctx, listErr}+ // Resolve each candidate's configured image once, not per record.+ candidateRefs := make(map[string]string, len(listed.Items))+ for _, candidate := range listed.Items {+ candidateRefs[candidate.ID] = refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, candidate))+ }for _, record := range records {- if !record.HasUpdate {- continue- }for _, candidate := range listed.Items {if excluded[candidate.ID] || (record.ContainerID != "" && record.ContainerID != candidate.ID) {continue}- if record.ContainerID == "" && refs.NormalizeImageUpdateRef(candidate.Image) != refs.NormalizeImageUpdateRef(record.ImageRef()) {+ if record.ContainerID == "" && candidateRefs[candidate.ID] != refs.NormalizeImageUpdateRef(record.ImageRef()) {continue}if deferred(record, candidate) {diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.goindex fc43a87bf..15bade45a 100644--- a/backend/internal/updater/service.go+++ b/backend/internal/updater/service.go@@ -219,6 +219,7 @@ func NewUpdaterService(UsedImageCollector: updater.UsedImageCollectorFunc(service.CollectUsedImages),LabelPolicy: updater.DefaultLabelPolicy(),SelfContainerID: kit.Ternary(selfErr != nil, "", selfContainerID),+ OperationTimeout: timeouts.DefaultAutoUpdateOperation,Logger: slog.Default(),})if err != nil {@@ -294,7 +295,7 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater}// applyBatch runs one update batch into out, records its events, and sends its notifications-// as one batch. The engine's per-container docker operations carry no timeouts, so the run is capped.+// as one batch, capped by a whole-run timeout.func (s *UpdaterService) applyBatch(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result) error {batch := &containerUpdateBatch{}ctx = context.WithValue(ctx, containerUpdateBatchContextKey{}, batch)@@ -391,13 +392,21 @@ func (s *UpdaterService) applyScopedUpdates(ctx context.Context, options arcaneuif listErr != nil {return nil, fmt.Errorf("list containers: %w", listErr)}+ matches := func(summary container.Summary) bool {+ return docker.ComposeServiceLabel(summary.Labels) != "" && wanted[strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))]+ }+ if scope == "image" {+ dockerClient, clientErr := s.DockerClient(ctx)+ if clientErr != nil {+ return nil, clientErr+ }+ matches = func(summary container.Summary) bool {+ return wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, summary))]+ }+ }var matched []stringfor _, summary := range containers {- if kit.Ternary(- scope == "project",- docker.ComposeServiceLabel(summary.Labels) != "" && wanted[strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))],- wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(summary.Image)],- ) {+ if matches(summary) {matched = append(matched, summary.ID)}}@@ -703,7 +712,12 @@ func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.Imaif err := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{}).Where("container_id <> ?", "").Limit(1).Count(&scopedCount).Error; err != nil {return nil, err}- running, err := func() ([]container.Summary, error) {+ // Resolve each container's policy and normalized configured ref once, not per record.+ type eligibleContainer struct {+ id string+ ref string+ }+ eligible, err := func() ([]eligibleContainer, error) {if scopedCount == 0 {return nil, nil}@@ -715,24 +729,20 @@ func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.Imaif listErr != nil {return nil, listErr}- return listed.Items, nil+ matched := make([]eligibleContainer, 0, len(listed.Items))+ for _, cnt := range listed.Items {+ configured := imageref.ConfiguredImage(ctx, dockerClient, cnt)+ resolved, policyErr := tagpolicy.Resolve(configured, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))+ if policyErr != nil || resolved.Strategy == "tag" {+ continue+ }+ matched = append(matched, eligibleContainer{id: cnt.ID, ref: refs.NormalizeImageUpdateRef(configured)})+ }+ return matched, nil}()if err != nil {return nil, err}- // Resolve each container's policy and normalized ref once, not per record.- type eligibleContainer struct {- id string- ref string- }- eligible := make([]eligibleContainer, 0, len(running))- for _, cnt := range running {- resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))- if policyErr != nil || resolved.Strategy == "tag" {- continue- }- eligible = append(eligible, eligibleContainer{id: cnt.ID, ref: refs.NormalizeImageUpdateRef(cnt.Image)})- }converted := func(record imageupdate.ImageUpdateRecord, containerID string) updater.ImageUpdateRecord {return updater.ImageUpdateRecord{ContainerID: containerID,@@ -790,11 +800,13 @@ func (s *UpdaterService) ClearImageUpdateRecord(ctx context.Context, record updareturn err}for _, cnt := range listed.Items {- resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))+ // Untagged images list by ID, so an orphaned container matches by its configured image.+ configured := imageref.ConfiguredImage(ctx, dockerClient, cnt)+ resolved, policyErr := tagpolicy.Resolve(configured, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))if policyErr != nil || resolved.Strategy == "tag" {continue}- if refs.NormalizeImageUpdateRef(cnt.Image) == refs.NormalizeImageUpdateRef(record.ImageRef()) && cnt.ImageID != target.ID {+ if refs.NormalizeImageUpdateRef(configured) == refs.NormalizeImageUpdateRef(record.ImageRef()) && cnt.ImageID != target.ID {return nil}}diff --git a/backend/pkg/libarcane/timeouts/timeouts.go b/backend/pkg/libarcane/timeouts/timeouts.goindex 15af6e5c6..6ffe197a1 100644--- a/backend/pkg/libarcane/timeouts/timeouts.go+++ b/backend/pkg/libarcane/timeouts/timeouts.go@@ -28,10 +28,12 @@ const (// per-environment concurrency slot before failing loudly instead of// parking forever behind hung work.DefaultActivitySlotWait = 2 * time.Minute+ // DefaultAutoUpdateOperation bounds each updater engine Docker mutation and+ // Compose project update. It must exceed DefaultDeployWait.+ DefaultAutoUpdateOperation = 15 * time.Minute// DefaultAutoUpdateApply bounds an entire auto-update (update-all) run.- // The updater engine's per-container docker operations carry no timeouts- // of their own, so this is the Arcane-side backstop that keeps a hung- // engine op from holding an activity slot indefinitely.+ // It is a whole-run backstop on top of the per-operation bounds, keeping+ // a long batch from holding an activity slot indefinitely.DefaultAutoUpdateApply = 30 * time.Minute)diff --git a/backend/pkg/utils/imageref/configured.go b/backend/pkg/utils/imageref/configured.gonew file mode 100644index 000000000..bb77c0fc5--- /dev/null+++ b/backend/pkg/utils/imageref/configured.go@@ -0,0 +1,26 @@+package imageref++import (+ "cmp"+ "context"++ "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/client"+ "go.getarcane.app/docker/compat"+ kit "go.getarcane.app/kit/pkg"+ "go.getarcane.app/updater/refs"+)++// ConfiguredImage returns the image a container was created from. Docker lists+// a container whose image lost its tag by image ID, so the configured image is+// inspected only then; the listed name is the fallback.+func ConfiguredImage(ctx context.Context, dockerClient *client.Client, summary container.Summary) string {+ if !refs.IsImageIDLikeReference(summary.Image) {+ return summary.Image+ }+ result, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, summary.ID, client.ContainerInspectOptions{})+ if err != nil {+ return summary.Image+ }+ return cmp.Or(kit.FromPtr(result.Container.Config).Image, summary.Image)+}