Skip to content

Modern Docker Management, Designed for Everyone

fix: update containers whose image tag was moved by an interrupted run

Kyle Mendell8928c098a978
diff --git a/backend/go.mod b/backend/go.mod
index 6d7ddf3fe..dbd0bd9b8 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -73,7 +73,7 @@ require (
go.getarcane.app/streams v0.4.6
go.getarcane.app/sys/cgroup v0.2.5
go.getarcane.app/sys/crypto v0.3.1
- go.getarcane.app/updater v0.11.8
+ go.getarcane.app/updater v0.11.9
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0
go.opentelemetry.io/contrib/exporters/autoexport v0.72.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
diff --git a/backend/go.sum b/backend/go.sum
index b406ad4bd..0c300a23f 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -771,8 +771,8 @@ go.getarcane.app/sys/cgroup v0.2.5 h1:iI1qURe+kOgTRN75sxkpPePjJwlxAx3c9e9kEwKAVn
go.getarcane.app/sys/cgroup v0.2.5/go.mod h1:hJyCjpx04VcaaSvkRsMvZl5OLUlAQ60K5gEy1ctuU4U=
go.getarcane.app/sys/crypto v0.3.1 h1:NzTk0IyeL8FKK71m+bL4WpMaSt+uLMZ7N+IWNFbpLnI=
go.getarcane.app/sys/crypto v0.3.1/go.mod h1:Rxnq1pb5HVSoJ4VKmL6LZGCrztMONaypXKc8Zg0BMhE=
-go.getarcane.app/updater v0.11.8 h1:skyIaDoUPXll10jdrZDoWDl2la/nBHZ1MECXtgiJ5xE=
-go.getarcane.app/updater v0.11.8/go.mod h1:kJy2PIstnWGZgvcWcwJeTTwYu6TvZl3APeLhvIoFR5I=
+go.getarcane.app/updater v0.11.9 h1:bi7SxRpq/QnP9OE5ZgJOHmoCOoSDi2sssXgMoo5Cqtc=
+go.getarcane.app/updater v0.11.9/go.mod h1:kJy2PIstnWGZgvcWcwJeTTwYu6TvZl3APeLhvIoFR5I=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
diff --git a/backend/internal/updater/children/recovery/service.go b/backend/internal/updater/children/recovery/service.go
index 3492a1d4e..6870953a7 100644
--- a/backend/internal/updater/children/recovery/service.go
+++ b/backend/internal/updater/children/recovery/service.go
@@ -18,6 +18,7 @@ import (
"go.getarcane.app/updater/refs"
"github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref"
)
// errNoFrozenPlan reports a run without a persisted frozen plan.
@@ -116,7 +117,7 @@ func (s *Service) FreezePending(ctx context.Context, run scheduler.Run, excluded
return ctx, err
}
records = slices.DeleteFunc(records, func(record updater.ImageUpdateRecord) bool {
- return excluded[refs.NormalizeImageUpdateRef(record.ImageRef())] || excluded[record.ContainerID]
+ return !record.HasUpdate || excluded[refs.NormalizeImageUpdateRef(record.ImageRef())] || excluded[record.ContainerID]
})
failures := map[string]error{}
selected := make([]updater.ImageUpdateRecord, 0, len(records))
@@ -130,15 +131,17 @@ func (s *Service) FreezePending(ctx context.Context, run scheduler.Run, excluded
if listErr != nil {
return ctx, listErr
}
+ // Resolve each candidate's configured image once, not per record.
+ candidateRefs := make(map[string]string, len(listed.Items))
+ for _, candidate := range listed.Items {
+ candidateRefs[candidate.ID] = refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, candidate))
+ }
for _, record := range records {
- if !record.HasUpdate {
- continue
- }
for _, candidate := range listed.Items {
if excluded[candidate.ID] || (record.ContainerID != "" && record.ContainerID != candidate.ID) {
continue
}
- if record.ContainerID == "" && refs.NormalizeImageUpdateRef(candidate.Image) != refs.NormalizeImageUpdateRef(record.ImageRef()) {
+ if record.ContainerID == "" && candidateRefs[candidate.ID] != refs.NormalizeImageUpdateRef(record.ImageRef()) {
continue
}
if deferred(record, candidate) {
diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.go
index fc43a87bf..15bade45a 100644
--- a/backend/internal/updater/service.go
+++ b/backend/internal/updater/service.go
@@ -219,6 +219,7 @@ func NewUpdaterService(
UsedImageCollector: updater.UsedImageCollectorFunc(service.CollectUsedImages),
LabelPolicy: updater.DefaultLabelPolicy(),
SelfContainerID: kit.Ternary(selfErr != nil, "", selfContainerID),
+ OperationTimeout: timeouts.DefaultAutoUpdateOperation,
Logger: slog.Default(),
})
if err != nil {
@@ -294,7 +295,7 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater
}
// applyBatch runs one update batch into out, records its events, and sends its notifications
-// as one batch. The engine's per-container docker operations carry no timeouts, so the run is capped.
+// as one batch, capped by a whole-run timeout.
func (s *UpdaterService) applyBatch(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result) error {
batch := &containerUpdateBatch{}
ctx = context.WithValue(ctx, containerUpdateBatchContextKey{}, batch)
@@ -391,13 +392,21 @@ func (s *UpdaterService) applyScopedUpdates(ctx context.Context, options arcaneu
if listErr != nil {
return nil, fmt.Errorf("list containers: %w", listErr)
}
+ matches := func(summary container.Summary) bool {
+ return docker.ComposeServiceLabel(summary.Labels) != "" && wanted[strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))]
+ }
+ if scope == "image" {
+ dockerClient, clientErr := s.DockerClient(ctx)
+ if clientErr != nil {
+ return nil, clientErr
+ }
+ matches = func(summary container.Summary) bool {
+ return wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, summary))]
+ }
+ }
var matched []string
for _, summary := range containers {
- if kit.Ternary(
- scope == "project",
- docker.ComposeServiceLabel(summary.Labels) != "" && wanted[strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels)))],
- wanted[summary.ImageID] || wanted[refs.NormalizeImageUpdateRef(summary.Image)],
- ) {
+ if matches(summary) {
matched = append(matched, summary.ID)
}
}
@@ -703,7 +712,12 @@ func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.Ima
if err := s.deps.DB.WithContext(ctx).Model(&imageupdate.ImageUpdateRecord{}).Where("container_id <> ?", "").Limit(1).Count(&scopedCount).Error; err != nil {
return nil, err
}
- running, err := func() ([]container.Summary, error) {
+ // Resolve each container's policy and normalized configured ref once, not per record.
+ type eligibleContainer struct {
+ id string
+ ref string
+ }
+ eligible, err := func() ([]eligibleContainer, error) {
if scopedCount == 0 {
return nil, nil
}
@@ -715,24 +729,20 @@ func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.Ima
if listErr != nil {
return nil, listErr
}
- return listed.Items, nil
+ matched := make([]eligibleContainer, 0, len(listed.Items))
+ for _, cnt := range listed.Items {
+ configured := imageref.ConfiguredImage(ctx, dockerClient, cnt)
+ resolved, policyErr := tagpolicy.Resolve(configured, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
+ if policyErr != nil || resolved.Strategy == "tag" {
+ continue
+ }
+ matched = append(matched, eligibleContainer{id: cnt.ID, ref: refs.NormalizeImageUpdateRef(configured)})
+ }
+ return matched, nil
}()
if err != nil {
return nil, err
}
- // Resolve each container's policy and normalized ref once, not per record.
- type eligibleContainer struct {
- id string
- ref string
- }
- eligible := make([]eligibleContainer, 0, len(running))
- for _, cnt := range running {
- resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
- if policyErr != nil || resolved.Strategy == "tag" {
- continue
- }
- eligible = append(eligible, eligibleContainer{id: cnt.ID, ref: refs.NormalizeImageUpdateRef(cnt.Image)})
- }
converted := func(record imageupdate.ImageUpdateRecord, containerID string) updater.ImageUpdateRecord {
return updater.ImageUpdateRecord{
ContainerID: containerID,
@@ -790,11 +800,13 @@ func (s *UpdaterService) ClearImageUpdateRecord(ctx context.Context, record upda
return err
}
for _, cnt := range listed.Items {
- resolved, policyErr := tagpolicy.Resolve(cnt.Image, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
+ // Untagged images list by ID, so an orphaned container matches by its configured image.
+ configured := imageref.ConfiguredImage(ctx, dockerClient, cnt)
+ resolved, policyErr := tagpolicy.Resolve(configured, updater.DefaultLabelPolicy().TagPolicy(cnt.Labels))
if policyErr != nil || resolved.Strategy == "tag" {
continue
}
- if refs.NormalizeImageUpdateRef(cnt.Image) == refs.NormalizeImageUpdateRef(record.ImageRef()) && cnt.ImageID != target.ID {
+ if refs.NormalizeImageUpdateRef(configured) == refs.NormalizeImageUpdateRef(record.ImageRef()) && cnt.ImageID != target.ID {
return nil
}
}
diff --git a/backend/pkg/libarcane/timeouts/timeouts.go b/backend/pkg/libarcane/timeouts/timeouts.go
index 15af6e5c6..6ffe197a1 100644
--- a/backend/pkg/libarcane/timeouts/timeouts.go
+++ b/backend/pkg/libarcane/timeouts/timeouts.go
@@ -28,10 +28,12 @@ const (
// per-environment concurrency slot before failing loudly instead of
// parking forever behind hung work.
DefaultActivitySlotWait = 2 * time.Minute
+ // DefaultAutoUpdateOperation bounds each updater engine Docker mutation and
+ // Compose project update. It must exceed DefaultDeployWait.
+ DefaultAutoUpdateOperation = 15 * time.Minute
// DefaultAutoUpdateApply bounds an entire auto-update (update-all) run.
- // The updater engine's per-container docker operations carry no timeouts
- // of their own, so this is the Arcane-side backstop that keeps a hung
- // engine op from holding an activity slot indefinitely.
+ // It is a whole-run backstop on top of the per-operation bounds, keeping
+ // a long batch from holding an activity slot indefinitely.
DefaultAutoUpdateApply = 30 * time.Minute
)
diff --git a/backend/pkg/utils/imageref/configured.go b/backend/pkg/utils/imageref/configured.go
new file mode 100644
index 000000000..bb77c0fc5
--- /dev/null
+++ b/backend/pkg/utils/imageref/configured.go
@@ -0,0 +1,26 @@
+package imageref
+
+import (
+ "cmp"
+ "context"
+
+ "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/client"
+ "go.getarcane.app/docker/compat"
+ kit "go.getarcane.app/kit/pkg"
+ "go.getarcane.app/updater/refs"
+)
+
+// ConfiguredImage returns the image a container was created from. Docker lists
+// a container whose image lost its tag by image ID, so the configured image is
+// inspected only then; the listed name is the fallback.
+func ConfiguredImage(ctx context.Context, dockerClient *client.Client, summary container.Summary) string {
+ if !refs.IsImageIDLikeReference(summary.Image) {
+ return summary.Image
+ }
+ result, err := compat.ContainerInspectWithCompatibility(ctx, dockerClient, summary.ID, client.ContainerInspectOptions{})
+ if err != nil {
+ return summary.Image
+ }
+ return cmp.Or(kit.FromPtr(result.Container.Config).Image, summary.Image)
+}