Skip to content

Modern Docker Management, Designed for Everyone

fix: lock updates per project instead of globally (#4520)

Kyle Mendellc8ab0db42b45
diff --git a/backend/internal/updater/admission.go b/backend/internal/updater/admission.go
new file mode 100644
index 000000000..15e6a38fa
--- /dev/null
+++ b/backend/internal/updater/admission.go
@@ -0,0 +1,195 @@
+package updater
+
+import (
+ "cmp"
+ "context"
+ "errors"
+ "fmt"
+ "maps"
+ "slices"
+ "strings"
+ "sync"
+
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/moby/moby/api/types/container"
+ "go.getarcane.app/docker"
+ kit "go.getarcane.app/kit/pkg"
+ "go.getarcane.app/updater/labels"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/common"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+)
+
+type updateLeases struct {
+ mu sync.Mutex
+ keys []scheduler.AdmissionKey
+ leases map[scheduler.AdmissionKey]*runs.Lease
+ retain map[scheduler.AdmissionKey]<-chan struct{}
+}
+
+func updateAdmissionKeys(containers []container.Summary) func(targetIDs []string) []scheduler.AdmissionKey {
+ identity := func(summary container.Summary) string {
+ if project := strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels))); project != "" {
+ return "project:" + project
+ }
+ ownNames := slices.DeleteFunc(slices.Clone(summary.Names), func(name string) bool {
+ return strings.Contains(strings.TrimPrefix(name, "/"), "/")
+ })
+ return "container:" + cmp.Or(docker.ContainerNameFromNames(ownNames), summary.ID)
+ }
+ lookup := func(ref string) (container.Summary, bool) {
+ if index := slices.IndexFunc(containers, func(summary container.Summary) bool {
+ return slices.ContainsFunc(summary.Names, func(name string) bool {
+ return strings.TrimPrefix(name, "/") == ref
+ })
+ }); index >= 0 {
+ return containers[index], true
+ }
+ index := slices.IndexFunc(containers, func(summary container.Summary) bool {
+ return ref != "" && strings.HasPrefix(summary.ID, ref)
+ })
+ if index < 0 {
+ return container.Summary{}, false
+ }
+ return containers[index], true
+ }
+
+ parent := map[string]string{}
+ var root func(id string) string
+ root = func(id string) string {
+ next, ok := parent[id]
+ if !ok || next == id {
+ parent[id] = id
+ return id
+ }
+ parent[id] = root(next)
+ return parent[id]
+ }
+ union := func(a, b string) {
+ parent[root(a)] = root(b)
+ }
+ for _, summary := range containers {
+ self := identity(summary)
+ root(self)
+ if strings.HasPrefix(self, "container:") {
+ union(self, "id:"+summary.ID)
+ }
+ networkMode := container.NetworkMode(summary.HostConfig.NetworkMode)
+ peers := kit.TrimNonEmpty(strings.Split(summary.Labels[labels.LabelDependsOn], ","))
+ if networkMode.IsContainer() {
+ peers = append(peers, networkMode.ConnectedContainer())
+ }
+ for _, name := range summary.Names {
+ if linker, _, linked := strings.Cut(strings.TrimPrefix(name, "/"), "/"); linked {
+ peers = append(peers, linker)
+ }
+ }
+ for _, ref := range peers {
+ if peer, found := lookup(ref); found {
+ union(self, identity(peer))
+ }
+ }
+ }
+
+ return func(targetIDs []string) []scheduler.AdmissionKey {
+ targetRoots := map[string]bool{}
+ for _, id := range kit.TrimNonEmpty(targetIDs) {
+ if project, isProject := strings.CutPrefix(id, "compose:"); isProject {
+ targetRoots[root("project:"+strings.ToLower(strings.TrimSpace(project)))] = true
+ continue
+ }
+ if summary, found := lookup(id); found {
+ targetRoots[root(identity(summary))] = true
+ continue
+ }
+ targetRoots[root("id:"+id)] = true
+ targetRoots[root("container:"+strings.TrimPrefix(id, "/"))] = true
+ }
+ identities := slices.Sorted(maps.Keys(parent))
+ keys := make([]scheduler.AdmissionKey, 0, len(identities))
+ for _, id := range identities {
+ if targetRoots[root(id)] {
+ keys = append(keys, scheduler.AdmissionKey{Scope: "updater", ID: id})
+ }
+ }
+ return keys
+ }
+}
+
+func (s *UpdaterService) acquireUpdate(ctx context.Context, containerIDs []string, wait bool) (context.Context, func(), error) {
+ if len(kit.TrimNonEmpty(containerIDs)) == 0 {
+ return s.admitUpdate(ctx, nil, wait)
+ }
+ containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)
+ if err != nil {
+ return ctx, nil, fmt.Errorf("list containers: %w", err)
+ }
+ return s.admitUpdate(ctx, updateAdmissionKeys(containers)(containerIDs), wait)
+}
+
+func (s *UpdaterService) admitUpdate(ctx context.Context, keys []scheduler.AdmissionKey, wait bool) (context.Context, func(), error) {
+ owner, nested := ctx.Value(updateAdmissionKey{}).(*updateLeases)
+ if !nested {
+ owner = &updateLeases{
+ leases: map[scheduler.AdmissionKey]*runs.Lease{},
+ retain: map[scheduler.AdmissionKey]<-chan struct{}{},
+ }
+ }
+ owner.mu.Lock()
+ defer owner.mu.Unlock()
+ keys = slices.DeleteFunc(slices.Clone(keys), func(key scheduler.AdmissionKey) bool {
+ return owner.leases[key] != nil
+ })
+ waitCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultActivitySlotWait)
+ defer cancel()
+ acquired := make([]*runs.Lease, 0, len(keys))
+ for _, key := range keys {
+ lease, acquireErr := func() (*runs.Lease, error) {
+ if wait {
+ return s.admission.Acquire(waitCtx, key)
+ }
+ lease, admitted, tryErr := s.admission.TryAcquire(ctx, key)
+ if tryErr == nil && !admitted {
+ return nil, errUpdateBusy
+ }
+ return lease, tryErr
+ }()
+ if acquireErr != nil {
+ for _, lease := range slices.Backward(acquired) {
+ lease.Release(ctx)
+ }
+ if ctx.Err() == nil && errors.Is(acquireErr, context.DeadlineExceeded) {
+ return ctx, nil, common.Classify(common.ErrTimeout, errors.New("timed out waiting for a conflicting container update to finish"))
+ }
+ return ctx, nil, acquireErr
+ }
+ acquired = append(acquired, lease)
+ }
+ for index, key := range keys {
+ owner.leases[key] = acquired[index]
+ }
+ owner.keys = append(owner.keys, keys...)
+ if nested {
+ return ctx, func() {}, nil
+ }
+ return context.WithValue(ctx, updateAdmissionKey{}, owner), func() {
+ owner.mu.Lock()
+ ordered := slices.Clone(owner.keys)
+ leases := maps.Clone(owner.leases)
+ retain := maps.Clone(owner.retain)
+ owner.mu.Unlock()
+ for _, key := range slices.Backward(ordered) {
+ lease := leases[key]
+ done, retained := retain[key]
+ if !retained {
+ lease.Release(ctx)
+ continue
+ }
+ go func() {
+ <-done
+ lease.Release(ctx)
+ }()
+ }
+ }, nil
+}
diff --git a/backend/internal/updater/admission_test.go b/backend/internal/updater/admission_test.go
new file mode 100644
index 000000000..a2708bb48
--- /dev/null
+++ b/backend/internal/updater/admission_test.go
@@ -0,0 +1,188 @@
+package updater
+
+import (
+ "context"
+ "encoding/json/v2"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/getarcaneapp/arcane/types/v2/scheduler"
+ "github.com/moby/moby/api/types/container"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+
+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"
+)
+
+const admissionTestContainers = `[
+ {"Id": "aaa111", "Names": ["/web-1"], "Labels": {"com.docker.compose.project": "Shop"}},
+ {"Id": "bbb222", "Names": ["/db-1"], "Labels": {"com.docker.compose.project": "shop"}},
+ {"Id": "ccc333", "Names": ["/blog-1"], "Labels": {"com.docker.compose.project": "blog"}},
+ {"Id": "ddd444", "Names": ["/vpn"]},
+ {"Id": "eee555", "Names": ["/torrent"], "HostConfig": {"NetworkMode": "container:vpn"}},
+ {"Id": "fff666", "Names": ["/indexer"], "HostConfig": {"NetworkMode": "container:ddd4"}},
+ {"Id": "ggg777", "Names": ["/worker"], "Labels": {"com.getarcaneapp.arcane.depends-on": "queue"}},
+ {"Id": "hhh888", "Names": ["/queue"]},
+ {"Id": "iii999", "Names": ["/solo"]},
+ {"Id": "jjj000", "Names": ["/app/cache", "/cache"]},
+ {"Id": "kkk111", "Names": ["/app"]}
+]`
+
+func admissionTestSummaries(t *testing.T) []container.Summary {
+ t.Helper()
+ var containers []container.Summary
+ require.NoError(t, json.Unmarshal([]byte(admissionTestContainers), &containers))
+ return containers
+}
+
+func newAdmissionTestService(t *testing.T) (*UpdaterService, *runs.Admission) {
+ t.Helper()
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !strings.HasSuffix(r.URL.Path, "/containers/json") {
+ http.NotFound(w, r)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(admissionTestContainers))
+ }))
+ t.Cleanup(server.Close)
+ db := setupProjectTestDB(t)
+ harness := flowtest.New(t, activity.NewActivityService(db, nil))
+ admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())
+ require.NoError(t, admission.Register(harness.Runtime))
+ svc, err := NewUpdaterService(
+ db,
+ nil,
+ (&docker.DockerClientService{}).WithClient(newTestDockerClient(t, server)),
+ nil,
+ nil,
+ nil,
+ nil,
+ nil,
+ nil,
+ nil,
+ nil,
+ nil,
+ admission,
+ nil,
+ )
+ require.NoError(t, err)
+ harness.Start(t)
+ return svc, admission
+}
+
+func TestUpdateAdmissionKeys(t *testing.T) {
+ containers := admissionTestSummaries(t)
+ keys := func(ids ...string) []scheduler.AdmissionKey {
+ out := make([]scheduler.AdmissionKey, 0, len(ids))
+ for _, id := range ids {
+ out = append(out, scheduler.AdmissionKey{Scope: "updater", ID: id})
+ }
+ return out
+ }
+ tests := []struct {
+ name string
+ targets []string
+ want []scheduler.AdmissionKey
+ }{
+ {name: "compose project members share the project key", targets: []string{"aaa111"}, want: keys("project:shop")},
+ {name: "separate projects stay separate", targets: []string{"ccc333"}, want: keys("project:blog")},
+ {
+ name: "network dependent joins its owner",
+ targets: []string{"eee555"},
+ want: keys("container:indexer", "container:torrent", "container:vpn", "id:ddd444", "id:eee555", "id:fff666"),
+ },
+ {
+ name: "network owner covers dependents by name and id prefix",
+ targets: []string{"ddd444"},
+ want: keys("container:indexer", "container:torrent", "container:vpn", "id:ddd444", "id:eee555", "id:fff666"),
+ },
+ {name: "depends-on label is unioned", targets: []string{"hhh888"}, want: keys("container:queue", "container:worker", "id:ggg777", "id:hhh888")},
+ {name: "legacy link target joins the linking container", targets: []string{"jjj000"}, want: keys("container:app", "container:cache", "id:jjj000", "id:kkk111")},
+ {name: "standalone container has its own key", targets: []string{"iii999"}, want: keys("container:solo", "id:iii999")},
+ {name: "missing container falls back to its id and name", targets: []string{"zzz000"}, want: keys("container:zzz000", "id:zzz000")},
+ {name: "compose ref locks a project with no containers", targets: []string{"compose:Gone"}, want: keys("project:gone")},
+ {name: "multiple targets are sorted and deduplicated", targets: []string{"bbb222", "iii999", "aaa111"}, want: keys("container:solo", "id:iii999", "project:shop")},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ assert.Equal(t, tt.want, updateAdmissionKeys(containers)(tt.targets))
+ })
+ }
+}
+
+func TestUpdaterService_AcquireUpdateUnrelatedKeyAdmitsImmediately(t *testing.T) {
+ svc, admission := newAdmissionTestService(t)
+ held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: "project:shop"})
+ require.NoError(t, err)
+ require.True(t, admitted)
+ defer held.Release(t.Context())
+
+ ctx, release, err := svc.acquireUpdate(t.Context(), []string{"iii999"}, false)
+ require.NoError(t, err)
+ defer release()
+
+ _, reentrantRelease, err := svc.acquireUpdate(ctx, []string{"iii999"}, false)
+ require.NoError(t, err)
+ reentrantRelease()
+}
+
+func TestUpdaterService_AcquireUpdateWaitsForConflictingKey(t *testing.T) {
+ svc, admission := newAdmissionTestService(t)
+ held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: "container:solo"})
+ require.NoError(t, err)
+ require.True(t, admitted)
+
+ _, _, err = svc.acquireUpdate(t.Context(), []string{"iii999"}, false)
+ require.ErrorIs(t, err, errUpdateBusy)
+
+ acquired := make(chan error, 1)
+ go func() {
+ _, release, acquireErr := svc.acquireUpdate(t.Context(), []string{"iii999"}, true)
+ if acquireErr == nil {
+ release()
+ }
+ acquired <- acquireErr
+ }()
+ select {
+ case acquireErr := <-acquired:
+ require.FailNow(t, "acquire did not wait for the conflicting update", "error: %v", acquireErr)
+ case <-time.After(400 * time.Millisecond):
+ }
+ held.Release(t.Context())
+ select {
+ case acquireErr := <-acquired:
+ require.NoError(t, acquireErr)
+ case <-time.After(5 * time.Second):
+ require.FailNow(t, "acquire was not admitted after release")
+ }
+}
+
+func TestUpdaterService_AcquireUpdateFailureReleasesHeldKeys(t *testing.T) {
+ svc, admission := newAdmissionTestService(t)
+ held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: "container:vpn"})
+ require.NoError(t, err)
+ require.True(t, admitted)
+ defer held.Release(t.Context())
+
+ _, _, err = svc.acquireUpdate(t.Context(), []string{"eee555"}, false)
+ require.ErrorIs(t, err, errUpdateBusy)
+
+ ctx, cancel := context.WithTimeout(t.Context(), 400*time.Millisecond)
+ defer cancel()
+ _, _, err = svc.acquireUpdate(ctx, []string{"eee555"}, true)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+
+ for _, id := range []string{"container:indexer", "container:torrent"} {
+ lease, free, tryErr := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: id})
+ require.NoError(t, tryErr)
+ require.True(t, free, id)
+ lease.Release(t.Context())
+ }
+}
diff --git a/backend/internal/updater/children/execution/service.go b/backend/internal/updater/children/execution/service.go
index 45f40f524..db810a995 100644
--- a/backend/internal/updater/children/execution/service.go
+++ b/backend/internal/updater/children/execution/service.go
@@ -31,7 +31,7 @@ type (
Dependencies struct {
Config *config.Config
Roles *role.RoleService
- AcquireUpdate func(ctx context.Context) (context.Context, func(), error)
+ AcquireUpdate func(ctx context.Context, ids []string) (context.Context, func(), error)
UpdateBusy error
RunUpdate func(ctx context.Context, containerID, activityID string) (*updater.Result, error)
// RecordUpdate saves the result on the activity and summarizes it as the run outcome.
@@ -106,7 +106,7 @@ func (s *Service) update(ctx context.Context, t flow.Task) (any, error) {
if err := t.Payload(&input); err != nil {
return nil, err
}
- ctx, release, err := s.deps.AcquireUpdate(ctx)
+ ctx, release, err := s.deps.AcquireUpdate(ctx, []string{input.ContainerID})
if errors.Is(err, s.deps.UpdateBusy) {
return nil, actor.ErrJobRejected
}
diff --git a/backend/internal/updater/children/recovery/service.go b/backend/internal/updater/children/recovery/service.go
index 73bf5b525..12237c469 100644
--- a/backend/internal/updater/children/recovery/service.go
+++ b/backend/internal/updater/children/recovery/service.go
@@ -5,6 +5,7 @@ import (
"context"
"encoding/json/v2"
"errors"
+ "fmt"
"slices"
"strings"
@@ -33,7 +34,7 @@ type Service struct {
images remoteDigester
pendingUpdates func(ctx context.Context) ([]updater.ImageUpdateRecord, error)
activityID func(ctx context.Context) string
- acquireUpdate func(ctx context.Context) (context.Context, func(), error)
+ acquireUpdate func(ctx context.Context, ids []string) (context.Context, func(), error)
applyPending func(ctx context.Context, options arcaneupdater.Options) (*arcaneupdater.Result, error)
}
@@ -61,7 +62,7 @@ func NewService(
images remoteDigester,
pendingUpdates func(ctx context.Context) ([]updater.ImageUpdateRecord, error),
activityID func(ctx context.Context) string,
- acquireUpdate func(ctx context.Context) (context.Context, func(), error),
+ acquireUpdate func(ctx context.Context, ids []string) (context.Context, func(), error),
applyPending func(ctx context.Context, options arcaneupdater.Options) (*arcaneupdater.Result, error),
) *Service {
return &Service{
@@ -439,7 +440,11 @@ func (s *Service) ConfirmTarget(ctx context.Context, target arcaneupdater.Frozen
// ReconcilePending resumes only frozen targets whose original container is unchanged.
func (s *Service) ReconcilePending(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {
- ctx, release, err := s.acquireUpdate(ctx)
+ targetIDs, err := s.TargetRefs(run)
+ if err != nil {
+ return scheduler.Outcome{Status: scheduler.Waiting}, err
+ }
+ ctx, release, err := s.acquireUpdate(ctx, targetIDs)
if err != nil {
return scheduler.Outcome{Status: scheduler.Waiting}, err
}
@@ -573,6 +578,21 @@ func (s *Service) Started(run scheduler.Run) bool {
})
}
+func (s *Service) TargetRefs(run scheduler.Run) ([]string, error) {
+ plan, _, err := frozenPlan(run)
+ if err != nil {
+ return nil, fmt.Errorf("decode frozen update plan: %w", err)
+ }
+ ids := make([]string, 0, len(plan.Targets)*3)
+ for _, target := range plan.Targets {
+ ids = append(ids, target.ContainerID, target.ContainerName)
+ if target.ComposeProject != "" {
+ ids = append(ids, "compose:"+target.ComposeProject)
+ }
+ }
+ return ids, nil
+}
+
// Unsettled counts the frozen targets a retry would still resume.
func (s *Service) Unsettled(run scheduler.Run) (int, error) {
plan, found, err := frozenPlan(run)
diff --git a/backend/internal/updater/children/recovery/service_test.go b/backend/internal/updater/children/recovery/service_test.go
index 98bd4a580..cbc15d502 100644
--- a/backend/internal/updater/children/recovery/service_test.go
+++ b/backend/internal/updater/children/recovery/service_test.go
@@ -44,7 +44,9 @@ func newTestService(t *testing.T, server *httptest.Server, pending func(context.
unresolvedDigests{},
pending,
func(context.Context) string { return "" },
- func(ctx context.Context) (context.Context, func(), error) { return ctx, func() {}, nil },
+ func(ctx context.Context, _ []string) (context.Context, func(), error) {
+ return ctx, func() {}, nil
+ },
nil,
)
}
diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.go
index fc141687d..236517cb4 100644
--- a/backend/internal/updater/service.go
+++ b/backend/internal/updater/service.go
@@ -90,9 +90,6 @@ type UpdaterService struct {
execution *execution.Service
recovery *recovery.Service
autoUpdate *flow.Workflow
- // updateMu serializes container updates: compose's recreate pipeline is not safe for
- // sibling containers sharing a namespace. Upgrade to per-project if throughput matters.
- updateMu sync.Mutex
}
type updaterDependencies struct {
@@ -194,14 +191,18 @@ func NewUpdaterService(
imageSvc,
service.PendingImageUpdates,
activityIDFromContext,
- service.acquireUpdate,
+ func(ctx context.Context, ids []string) (context.Context, func(), error) {
+ return service.acquireUpdate(ctx, ids, false)
+ },
service.ApplyPending,
)
// keep: execution callbacks bind to the allocated service and its recovery
service.execution = execution.NewService(execution.Dependencies{
- Config: cfg,
- Roles: roles,
- AcquireUpdate: service.acquireUpdate,
+ Config: cfg,
+ Roles: roles,
+ AcquireUpdate: func(ctx context.Context, ids []string) (context.Context, func(), error) {
+ return service.acquireUpdate(ctx, ids, false)
+ },
UpdateBusy: errUpdateBusy,
RunUpdate: service.runSingleContainerUpdate,
RecordUpdate: service.recordSingleContainerUpdate,
@@ -242,8 +243,12 @@ func NewUpdaterService(
// ApplyPending executes pending image updates. A project or image scope limits the engine's
// pending records to its containers; a container scope or a forced run updates each container.
func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater.Options) (out *arcaneupdater.Result, err error) {
+ containerIDs, err := s.pendingContainerIDs(ctx, options)
+ if err != nil {
+ return nil, err
+ }
var release func()
- ctx, release, err = s.acquireUpdate(ctx)
+ ctx, release, err = s.acquireUpdate(ctx, containerIDs, true)
if err != nil {
return nil, err
}
@@ -370,6 +375,46 @@ func (s *UpdaterService) applyBatch(ctx context.Context, options arcaneupdater.O
return nil
}
+func (s *UpdaterService) pendingContainerIDs(ctx context.Context, options arcaneupdater.Options) ([]string, error) {
+ if len(options.ResourceIds) > 0 {
+ containerIDs, _, _, _, err := s.scopedContainerIDs(ctx, options)
+ return containerIDs, err
+ }
+ if frozen, ok := s.recovery.FrozenRecords(ctx); ok {
+ return s.recordContainerIDs(ctx, frozen)
+ }
+ var records []imageupdate.ImageUpdateRecord
+ if err := s.deps.DB.WithContext(ctx).Select("container_id", "repository", "tag").Where("has_update = ? AND project_id = ?", true, "").Find(&records).Error; err != nil {
+ return nil, fmt.Errorf("query pending image updates: %w", err)
+ }
+ pending := make([]updater.ImageUpdateRecord, 0, len(records))
+ for _, record := range records {
+ pending = append(pending, updater.ImageUpdateRecord{
+ ContainerID: record.ContainerID,
+ Repository: record.Repository,
+ Tag: record.Tag,
+ })
+ }
+ return s.recordContainerIDs(ctx, pending)
+}
+
+func (s *UpdaterService) recordContainerIDs(ctx context.Context, records []updater.ImageUpdateRecord) ([]string, error) {
+ containerIDs := make([]string, 0, len(records))
+ imageRefs := make([]string, 0, len(records))
+ for _, record := range records {
+ if record.ContainerID != "" {
+ containerIDs = append(containerIDs, record.ContainerID)
+ continue
+ }
+ imageRefs = append(imageRefs, record.ImageRef())
+ }
+ imageContainerIDs, _, _, _, err := s.scopedContainerIDs(ctx, arcaneupdater.Options{Type: "image", ResourceIds: imageRefs})
+ if err != nil {
+ return nil, err
+ }
+ return kit.Unique(kit.TrimNonEmpty(slices.Concat(containerIDs, imageContainerIDs))), nil
+}
+
// scopedContainerIDs resolves the requested containers, projects, or images to container IDs.
func (s *UpdaterService) scopedContainerIDs(ctx context.Context, options arcaneupdater.Options) ([]string, map[string]string, map[string]string, map[string]bool, error) {
requested := kit.TrimNonEmpty(options.ResourceIds)
@@ -645,7 +690,7 @@ func (s *UpdaterService) AcceptSingleContainerUpdate(ctx context.Context, contai
func (s *UpdaterService) runSingleContainerUpdate(ctx context.Context, containerID, activityID string) (out *arcaneupdater.Result, err error) {
var release func()
- ctx, release, err = s.acquireUpdate(ctx)
+ ctx, release, err = s.acquireUpdate(ctx, []string{containerID}, true)
if err != nil {
return nil, err
}
@@ -665,8 +710,6 @@ func (s *UpdaterService) runSingleContainerUpdate(ctx context.Context, container
}
}
- s.updateMu.Lock()
- defer s.updateMu.Unlock()
if ctxErr := ctx.Err(); ctxErr != nil {
return out, ctxErr
}
@@ -797,6 +840,68 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progres
// PendingImageUpdates returns pending image update records from Arcane's database.
func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.ImageUpdateRecord, error) {
+ records, err := s.loadPendingImageUpdates(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if _, batch := ctx.Value(updateAdmissionKey{}).(*updateLeases); !batch || len(records) == 0 {
+ return records, nil
+ }
+ containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("list containers: %w", err)
+ }
+ dockerClient, err := s.deps.Docker.GetClient(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("connect to docker: %w", err)
+ }
+ keysFor := updateAdmissionKeys(containers)
+ configured := map[string]string{}
+ if slices.ContainsFunc(records, func(record updater.ImageUpdateRecord) bool { return record.ContainerID == "" }) {
+ for _, summary := range containers {
+ configured[summary.ID] = refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, summary))
+ }
+ }
+ admitted := make([]updater.ImageUpdateRecord, 0, len(records))
+ for _, record := range records {
+ containerIDs, matchErr := func() ([]string, error) {
+ if record.ContainerID != "" {
+ return []string{record.ContainerID}, nil
+ }
+ if record.ImageRef() == "" {
+ return nil, nil
+ }
+ inspected, inspectErr := dockerClient.ImageInspect(ctx, record.ImageRef())
+ if inspectErr != nil && !errdefs.IsNotFound(inspectErr) {
+ return nil, fmt.Errorf("inspect pending image %s: %w", record.ImageRef(), inspectErr)
+ }
+ oldIDs := kit.TrimNonEmpty([]string{inspected.ID, record.ID})
+ normalized := refs.NormalizeImageUpdateRef(record.ImageRef())
+ var matched []string
+ for _, summary := range containers {
+ if slices.Contains(oldIDs, summary.ImageID) || (normalized != "" && configured[summary.ID] == normalized) {
+ matched = append(matched, summary.ID)
+ }
+ }
+ return matched, nil
+ }()
+ if matchErr != nil {
+ return nil, matchErr
+ }
+ _, _, acquireErr := s.admitUpdate(ctx, keysFor(containerIDs), false)
+ if errors.Is(acquireErr, errUpdateBusy) {
+ slog.InfoContext(ctx, "skipping pending update for a container another update holds", "imageRef", record.ImageRef(), "containerId", record.ContainerID)
+ continue
+ }
+ if acquireErr != nil {
+ return nil, acquireErr
+ }
+ admitted = append(admitted, record)
+ }
+ return admitted, nil
+}
+
+func (s *UpdaterService) loadPendingImageUpdates(ctx context.Context) ([]updater.ImageUpdateRecord, error) {
scoped, isScoped := ctx.Value(scopedPendingKey{}).(map[string]string)
outOfScope := func(record updater.ImageUpdateRecord) bool {
_, inScope := scoped[record.ContainerID]
@@ -1002,10 +1107,33 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.S
if pullErr != nil {
return pullErr
}
+ containers, _, _, _, listErr := s.deps.Docker.GetAllContainers(ctx)
+ if listErr != nil {
+ return fmt.Errorf("list containers: %w", listErr)
+ }
+ stopsProcess := target.InstanceType != "agent"
+ keys := updateAdmissionKeys(containers)(func() []string {
+ if !stopsProcess {
+ return []string{target.ContainerID}
+ }
+ allIDs := make([]string, 0, len(containers))
+ for _, summary := range containers {
+ allIDs = append(allIDs, summary.ID)
+ }
+ return allIDs
+ }())
+ ctx, release, acquireErr := s.admitUpdate(ctx, keys, false)
+ if errors.Is(acquireErr, errUpdateBusy) {
+ return common.Classify(common.ErrConflict, errors.New("another container update is in progress; the self-update stays pending for the next run"))
+ }
+ if acquireErr != nil {
+ return acquireErr
+ }
+ defer release()
// A server self-update stops this process before the run completes, so the
// activity is flagged first and startup reconciliation finalizes it.
- if activityID := activityIDFromContext(ctx); target.InstanceType != "agent" && activityID != "" {
+ if activityID := activityIDFromContext(ctx); stopsProcess && activityID != "" {
s.appendAutoUpdateActivityMessage(
ctx,
activityID,
@@ -1018,16 +1146,49 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.S
}
}
- if _, err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, updater.SelfUpdateTarget{ //exhaustruct:enforce
+ upgraderID, err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, updater.SelfUpdateTarget{ //exhaustruct:enforce
ContainerID: target.ContainerID,
ContainerName: target.ContainerName,
InstanceType: target.InstanceType,
Labels: target.Labels,
NewImageRef: target.NewImageRef,
PullImageRef: pullImageRef,
- }); err != nil {
+ })
+ if err != nil {
return fmt.Errorf("CLI upgrade failed: %w", err)
}
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ waitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDeployWait)
+ defer cancel()
+ if stopsProcess || upgraderID == "" {
+ <-waitCtx.Done()
+ return
+ }
+ dockerClient, clientErr := s.deps.Docker.GetClient(waitCtx)
+ if clientErr != nil {
+ slog.WarnContext(waitCtx, "cannot watch agent upgrader; holding its update keys until the deploy wait ends", "upgraderId", upgraderID, "error", clientErr)
+ <-waitCtx.Done()
+ return
+ }
+ wait := dockerClient.ContainerWait(waitCtx, upgraderID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})
+ select {
+ case <-wait.Result:
+ case waitErr := <-wait.Error:
+ if !errdefs.IsNotFound(waitErr) {
+ slog.WarnContext(waitCtx, "agent upgrader wait failed; holding its update keys until the deploy wait ends", "upgraderId", upgraderID, "error", waitErr)
+ <-waitCtx.Done()
+ }
+ }
+ }()
+ if leases, ok := ctx.Value(updateAdmissionKey{}).(*updateLeases); ok {
+ leases.mu.Lock()
+ for _, key := range kit.Ternary(stopsProcess, leases.keys, keys) {
+ leases.retain[key] = done
+ }
+ leases.mu.Unlock()
+ }
if progress, ok := ctx.Value(updateProgressKey{}).(*updateProgress); ok {
progress.mu.Lock()
progress.selfTriggered = true
@@ -1447,20 +1608,6 @@ func addNormalizedImageUpdateRef(ctx context.Context, out map[string]struct{}, i
slog.DebugContext(ctx, "skipping invalid image reference", append(attrs, "source", source, "imageRef", imageRef)...)
}
-func (s *UpdaterService) acquireUpdate(ctx context.Context) (context.Context, func(), error) {
- if ctx.Value(updateAdmissionKey{}) == s {
- return ctx, func() {}, nil
- }
- lease, admitted, err := s.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: "updater"})
- if err != nil {
- return ctx, nil, err
- }
- if !admitted {
- return ctx, nil, errUpdateBusy
- }
- return context.WithValue(ctx, updateAdmissionKey{}, s), func() { lease.Release(ctx) }, nil
-}
-
func (s *UpdaterService) PruneHistory(ctx context.Context, retentionDays int) (int64, error) {
if retentionDays <= 0 {
return 0, nil
@@ -1873,7 +2020,11 @@ func (s *UpdaterService) applyAutoUpdate(ctx context.Context, t flow.Task) (any,
if !ok {
return nil, errors.New("auto-update requires a job run")
}
- ctx, release, err := s.acquireUpdate(ctx)
+ targetIDs, err := s.recovery.TargetRefs(run)
+ if err != nil {
+ return nil, common.Classify(common.ErrUnavailable, err)
+ }
+ ctx, release, err := s.acquireUpdate(ctx, targetIDs, false)
if errors.Is(err, errUpdateBusy) {
// A plan whose every target already settled needs no updater, so it finishes normally.
if unsettled, unsettledErr := s.recovery.Unsettled(run); unsettledErr == nil && unsettled == 0 {
diff --git a/backend/internal/updater/service_test.go b/backend/internal/updater/service_test.go
index 2aa675d0f..5724f4d1f 100644
--- a/backend/internal/updater/service_test.go
+++ b/backend/internal/updater/service_test.go
@@ -332,10 +332,18 @@ func TestUpdaterService_ResultFromModulePreservesRestarted(t *testing.T) {
func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
ctx := t.Context()
+ newAdmission := func(t *testing.T) *runs.Admission {
+ t.Helper()
+ harness := flowtest.New(t, activity.NewActivityService(setupProjectTestDB(t), nil))
+ admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())
+ require.NoError(t, admission.Register(harness.Runtime))
+ harness.Start(t)
+ return admission
+ }
t.Run("server label triggers upgrade with system user", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{
@@ -356,7 +364,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("engine target keeps the selected tag and pulls the frozen digest", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)
require.NoError(t, svcErr)
frozen := svc.recovery.WithSingleTarget(ctx, &arcaneupdater.FrozenUpdateTarget{
@@ -379,7 +387,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("agent label triggers upgrade", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane-agent", map[string]string{
@@ -392,7 +400,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("non Arcane labels fail without triggering upgrade", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "demo", map[string]string{
@@ -406,7 +414,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {
t.Run("upgrade errors are wrapped", func(t *testing.T) {
mockUpgrade := &mockSystemUpgradeService{triggerError: errors.New("upgrade failed")}
- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)
+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)
require.NoError(t, svcErr)
err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{
@@ -863,13 +871,13 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivity(t *testing
harness := flowtest.New(t, activityService)
admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())
require.NoError(t, admission.Register(harness.Runtime))
- svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, admission, nil)
+ svc, err := NewUpdaterService(db, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, nil, activityService, nil, admission, nil)
require.NoError(t, err)
require.NoError(t, svc.execution.RegisterWorkflows(harness.Engine))
harness.Start(t)
// Holding the update admission makes the workflow hand its task back until the lease is released.
- held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater"})
+ held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: "id:missing-container"})
require.NoError(t, err)
require.True(t, admitted)
var releaseOnce sync.Once
@@ -1293,7 +1301,7 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgrade(
svc, svcErr := NewUpdaterService(
db,
nil,
- nil,
+ (&docker.DockerClientService{}).WithClient(newTestDockerClient(t, server)),
nil,
imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil),
nil,
diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.go
index 4009abb70..5bfaf8639 100644
--- a/backend/pkg/scheduler/runs/actors_test.go
+++ b/backend/pkg/scheduler/runs/actors_test.go
@@ -319,6 +319,44 @@ func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {
require.Equal(t, "new-token", persisted.Owner)
}
+func TestAdmissionAcquireWaitsForRelease(t *testing.T) {
+ host := francistest.New(t)
+ admission := NewAdmission(host.Service(), "epoch")
+ require.NoError(t, admission.Register(host))
+ francistest.Start(t, host)
+ key := st.AdmissionKey{Scope: "updater", ID: "container:solo"}
+ held, admitted, err := admission.TryAcquire(t.Context(), key)
+ require.NoError(t, err)
+ require.True(t, admitted)
+
+ canceledCtx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond)
+ defer cancel()
+ _, err = admission.Acquire(canceledCtx, key)
+ require.ErrorIs(t, err, context.DeadlineExceeded)
+
+ acquired := make(chan *Lease, 1)
+ go func() {
+ lease, acquireErr := admission.Acquire(t.Context(), key)
+ if acquireErr != nil {
+ lease = nil
+ }
+ acquired <- lease
+ }()
+ select {
+ case <-acquired:
+ require.FailNow(t, "acquire returned while the key was held")
+ case <-time.After(400 * time.Millisecond):
+ }
+ held.Release(t.Context())
+ select {
+ case lease := <-acquired:
+ require.NotNil(t, lease)
+ lease.Release(t.Context())
+ case <-time.After(5 * time.Second):
+ require.FailNow(t, "acquire was not admitted after release")
+ }
+}
+
func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {
q := newResolutionQueue(t)
entered := make(chan struct{})
diff --git a/backend/pkg/scheduler/runs/admission.go b/backend/pkg/scheduler/runs/admission.go
index 73a2d9158..64e50c75b 100644
--- a/backend/pkg/scheduler/runs/admission.go
+++ b/backend/pkg/scheduler/runs/admission.go
@@ -72,6 +72,22 @@ func (a *Admission) TryAcquire(ctx context.Context, key st.AdmissionKey) (*Lease
return &Lease{admission: a, id: id, token: token}, true, nil
}
+func (a *Admission) Acquire(ctx context.Context, key st.AdmissionKey) (*Lease, error) {
+ ticker := time.NewTicker(250 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ lease, admitted, err := a.TryAcquire(ctx, key)
+ if err != nil || admitted {
+ return lease, err
+ }
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-ticker.C:
+ }
+ }
+}
+
func (l *Lease) Release(ctx context.Context) {
if l == nil {
return