Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: lock updates per project instead of globally (#4520)
Kyle Mendellc8ab0db42b45
diff --git a/backend/internal/updater/admission.go b/backend/internal/updater/admission.gonew file mode 100644index 000000000..15e6a38fa--- /dev/null+++ b/backend/internal/updater/admission.go@@ -0,0 +1,195 @@+package updater++import (+ "cmp"+ "context"+ "errors"+ "fmt"+ "maps"+ "slices"+ "strings"+ "sync"++ "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/moby/moby/api/types/container"+ "go.getarcane.app/docker"+ kit "go.getarcane.app/kit/pkg"+ "go.getarcane.app/updater/labels"++ "github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+)++type updateLeases struct {+ mu sync.Mutex+ keys []scheduler.AdmissionKey+ leases map[scheduler.AdmissionKey]*runs.Lease+ retain map[scheduler.AdmissionKey]<-chan struct{}+}++func updateAdmissionKeys(containers []container.Summary) func(targetIDs []string) []scheduler.AdmissionKey {+ identity := func(summary container.Summary) string {+ if project := strings.ToLower(strings.TrimSpace(docker.ComposeProjectLabel(summary.Labels))); project != "" {+ return "project:" + project+ }+ ownNames := slices.DeleteFunc(slices.Clone(summary.Names), func(name string) bool {+ return strings.Contains(strings.TrimPrefix(name, "/"), "/")+ })+ return "container:" + cmp.Or(docker.ContainerNameFromNames(ownNames), summary.ID)+ }+ lookup := func(ref string) (container.Summary, bool) {+ if index := slices.IndexFunc(containers, func(summary container.Summary) bool {+ return slices.ContainsFunc(summary.Names, func(name string) bool {+ return strings.TrimPrefix(name, "/") == ref+ })+ }); index >= 0 {+ return containers[index], true+ }+ index := slices.IndexFunc(containers, func(summary container.Summary) bool {+ return ref != "" && strings.HasPrefix(summary.ID, ref)+ })+ if index < 0 {+ return container.Summary{}, false+ }+ return containers[index], true+ }++ parent := map[string]string{}+ var root func(id string) string+ root = func(id string) string {+ next, ok := parent[id]+ if !ok || next == id {+ parent[id] = id+ return id+ }+ parent[id] = root(next)+ return parent[id]+ }+ union := func(a, b string) {+ parent[root(a)] = root(b)+ }+ for _, summary := range containers {+ self := identity(summary)+ root(self)+ if strings.HasPrefix(self, "container:") {+ union(self, "id:"+summary.ID)+ }+ networkMode := container.NetworkMode(summary.HostConfig.NetworkMode)+ peers := kit.TrimNonEmpty(strings.Split(summary.Labels[labels.LabelDependsOn], ","))+ if networkMode.IsContainer() {+ peers = append(peers, networkMode.ConnectedContainer())+ }+ for _, name := range summary.Names {+ if linker, _, linked := strings.Cut(strings.TrimPrefix(name, "/"), "/"); linked {+ peers = append(peers, linker)+ }+ }+ for _, ref := range peers {+ if peer, found := lookup(ref); found {+ union(self, identity(peer))+ }+ }+ }++ return func(targetIDs []string) []scheduler.AdmissionKey {+ targetRoots := map[string]bool{}+ for _, id := range kit.TrimNonEmpty(targetIDs) {+ if project, isProject := strings.CutPrefix(id, "compose:"); isProject {+ targetRoots[root("project:"+strings.ToLower(strings.TrimSpace(project)))] = true+ continue+ }+ if summary, found := lookup(id); found {+ targetRoots[root(identity(summary))] = true+ continue+ }+ targetRoots[root("id:"+id)] = true+ targetRoots[root("container:"+strings.TrimPrefix(id, "/"))] = true+ }+ identities := slices.Sorted(maps.Keys(parent))+ keys := make([]scheduler.AdmissionKey, 0, len(identities))+ for _, id := range identities {+ if targetRoots[root(id)] {+ keys = append(keys, scheduler.AdmissionKey{Scope: "updater", ID: id})+ }+ }+ return keys+ }+}++func (s *UpdaterService) acquireUpdate(ctx context.Context, containerIDs []string, wait bool) (context.Context, func(), error) {+ if len(kit.TrimNonEmpty(containerIDs)) == 0 {+ return s.admitUpdate(ctx, nil, wait)+ }+ containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)+ if err != nil {+ return ctx, nil, fmt.Errorf("list containers: %w", err)+ }+ return s.admitUpdate(ctx, updateAdmissionKeys(containers)(containerIDs), wait)+}++func (s *UpdaterService) admitUpdate(ctx context.Context, keys []scheduler.AdmissionKey, wait bool) (context.Context, func(), error) {+ owner, nested := ctx.Value(updateAdmissionKey{}).(*updateLeases)+ if !nested {+ owner = &updateLeases{+ leases: map[scheduler.AdmissionKey]*runs.Lease{},+ retain: map[scheduler.AdmissionKey]<-chan struct{}{},+ }+ }+ owner.mu.Lock()+ defer owner.mu.Unlock()+ keys = slices.DeleteFunc(slices.Clone(keys), func(key scheduler.AdmissionKey) bool {+ return owner.leases[key] != nil+ })+ waitCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultActivitySlotWait)+ defer cancel()+ acquired := make([]*runs.Lease, 0, len(keys))+ for _, key := range keys {+ lease, acquireErr := func() (*runs.Lease, error) {+ if wait {+ return s.admission.Acquire(waitCtx, key)+ }+ lease, admitted, tryErr := s.admission.TryAcquire(ctx, key)+ if tryErr == nil && !admitted {+ return nil, errUpdateBusy+ }+ return lease, tryErr+ }()+ if acquireErr != nil {+ for _, lease := range slices.Backward(acquired) {+ lease.Release(ctx)+ }+ if ctx.Err() == nil && errors.Is(acquireErr, context.DeadlineExceeded) {+ return ctx, nil, common.Classify(common.ErrTimeout, errors.New("timed out waiting for a conflicting container update to finish"))+ }+ return ctx, nil, acquireErr+ }+ acquired = append(acquired, lease)+ }+ for index, key := range keys {+ owner.leases[key] = acquired[index]+ }+ owner.keys = append(owner.keys, keys...)+ if nested {+ return ctx, func() {}, nil+ }+ return context.WithValue(ctx, updateAdmissionKey{}, owner), func() {+ owner.mu.Lock()+ ordered := slices.Clone(owner.keys)+ leases := maps.Clone(owner.leases)+ retain := maps.Clone(owner.retain)+ owner.mu.Unlock()+ for _, key := range slices.Backward(ordered) {+ lease := leases[key]+ done, retained := retain[key]+ if !retained {+ lease.Release(ctx)+ continue+ }+ go func() {+ <-done+ lease.Release(ctx)+ }()+ }+ }, nil+}diff --git a/backend/internal/updater/admission_test.go b/backend/internal/updater/admission_test.gonew file mode 100644index 000000000..a2708bb48--- /dev/null+++ b/backend/internal/updater/admission_test.go@@ -0,0 +1,188 @@+package updater++import (+ "context"+ "encoding/json/v2"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/moby/moby/api/types/container"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"++ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+)++const admissionTestContainers = `[+ {"Id": "aaa111", "Names": ["/web-1"], "Labels": {"com.docker.compose.project": "Shop"}},+ {"Id": "bbb222", "Names": ["/db-1"], "Labels": {"com.docker.compose.project": "shop"}},+ {"Id": "ccc333", "Names": ["/blog-1"], "Labels": {"com.docker.compose.project": "blog"}},+ {"Id": "ddd444", "Names": ["/vpn"]},+ {"Id": "eee555", "Names": ["/torrent"], "HostConfig": {"NetworkMode": "container:vpn"}},+ {"Id": "fff666", "Names": ["/indexer"], "HostConfig": {"NetworkMode": "container:ddd4"}},+ {"Id": "ggg777", "Names": ["/worker"], "Labels": {"com.getarcaneapp.arcane.depends-on": "queue"}},+ {"Id": "hhh888", "Names": ["/queue"]},+ {"Id": "iii999", "Names": ["/solo"]},+ {"Id": "jjj000", "Names": ["/app/cache", "/cache"]},+ {"Id": "kkk111", "Names": ["/app"]}+]`++func admissionTestSummaries(t *testing.T) []container.Summary {+ t.Helper()+ var containers []container.Summary+ require.NoError(t, json.Unmarshal([]byte(admissionTestContainers), &containers))+ return containers+}++func newAdmissionTestService(t *testing.T) (*UpdaterService, *runs.Admission) {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if !strings.HasSuffix(r.URL.Path, "/containers/json") {+ http.NotFound(w, r)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(admissionTestContainers))+ }))+ t.Cleanup(server.Close)+ db := setupProjectTestDB(t)+ harness := flowtest.New(t, activity.NewActivityService(db, nil))+ admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())+ require.NoError(t, admission.Register(harness.Runtime))+ svc, err := NewUpdaterService(+ db,+ nil,+ (&docker.DockerClientService{}).WithClient(newTestDockerClient(t, server)),+ nil,+ nil,+ nil,+ nil,+ nil,+ nil,+ nil,+ nil,+ nil,+ admission,+ nil,+ )+ require.NoError(t, err)+ harness.Start(t)+ return svc, admission+}++func TestUpdateAdmissionKeys(t *testing.T) {+ containers := admissionTestSummaries(t)+ keys := func(ids ...string) []scheduler.AdmissionKey {+ out := make([]scheduler.AdmissionKey, 0, len(ids))+ for _, id := range ids {+ out = append(out, scheduler.AdmissionKey{Scope: "updater", ID: id})+ }+ return out+ }+ tests := []struct {+ name string+ targets []string+ want []scheduler.AdmissionKey+ }{+ {name: "compose project members share the project key", targets: []string{"aaa111"}, want: keys("project:shop")},+ {name: "separate projects stay separate", targets: []string{"ccc333"}, want: keys("project:blog")},+ {+ name: "network dependent joins its owner",+ targets: []string{"eee555"},+ want: keys("container:indexer", "container:torrent", "container:vpn", "id:ddd444", "id:eee555", "id:fff666"),+ },+ {+ name: "network owner covers dependents by name and id prefix",+ targets: []string{"ddd444"},+ want: keys("container:indexer", "container:torrent", "container:vpn", "id:ddd444", "id:eee555", "id:fff666"),+ },+ {name: "depends-on label is unioned", targets: []string{"hhh888"}, want: keys("container:queue", "container:worker", "id:ggg777", "id:hhh888")},+ {name: "legacy link target joins the linking container", targets: []string{"jjj000"}, want: keys("container:app", "container:cache", "id:jjj000", "id:kkk111")},+ {name: "standalone container has its own key", targets: []string{"iii999"}, want: keys("container:solo", "id:iii999")},+ {name: "missing container falls back to its id and name", targets: []string{"zzz000"}, want: keys("container:zzz000", "id:zzz000")},+ {name: "compose ref locks a project with no containers", targets: []string{"compose:Gone"}, want: keys("project:gone")},+ {name: "multiple targets are sorted and deduplicated", targets: []string{"bbb222", "iii999", "aaa111"}, want: keys("container:solo", "id:iii999", "project:shop")},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ assert.Equal(t, tt.want, updateAdmissionKeys(containers)(tt.targets))+ })+ }+}++func TestUpdaterService_AcquireUpdateUnrelatedKeyAdmitsImmediately(t *testing.T) {+ svc, admission := newAdmissionTestService(t)+ held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: "project:shop"})+ require.NoError(t, err)+ require.True(t, admitted)+ defer held.Release(t.Context())++ ctx, release, err := svc.acquireUpdate(t.Context(), []string{"iii999"}, false)+ require.NoError(t, err)+ defer release()++ _, reentrantRelease, err := svc.acquireUpdate(ctx, []string{"iii999"}, false)+ require.NoError(t, err)+ reentrantRelease()+}++func TestUpdaterService_AcquireUpdateWaitsForConflictingKey(t *testing.T) {+ svc, admission := newAdmissionTestService(t)+ held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: "container:solo"})+ require.NoError(t, err)+ require.True(t, admitted)++ _, _, err = svc.acquireUpdate(t.Context(), []string{"iii999"}, false)+ require.ErrorIs(t, err, errUpdateBusy)++ acquired := make(chan error, 1)+ go func() {+ _, release, acquireErr := svc.acquireUpdate(t.Context(), []string{"iii999"}, true)+ if acquireErr == nil {+ release()+ }+ acquired <- acquireErr+ }()+ select {+ case acquireErr := <-acquired:+ require.FailNow(t, "acquire did not wait for the conflicting update", "error: %v", acquireErr)+ case <-time.After(400 * time.Millisecond):+ }+ held.Release(t.Context())+ select {+ case acquireErr := <-acquired:+ require.NoError(t, acquireErr)+ case <-time.After(5 * time.Second):+ require.FailNow(t, "acquire was not admitted after release")+ }+}++func TestUpdaterService_AcquireUpdateFailureReleasesHeldKeys(t *testing.T) {+ svc, admission := newAdmissionTestService(t)+ held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: "container:vpn"})+ require.NoError(t, err)+ require.True(t, admitted)+ defer held.Release(t.Context())++ _, _, err = svc.acquireUpdate(t.Context(), []string{"eee555"}, false)+ require.ErrorIs(t, err, errUpdateBusy)++ ctx, cancel := context.WithTimeout(t.Context(), 400*time.Millisecond)+ defer cancel()+ _, _, err = svc.acquireUpdate(ctx, []string{"eee555"}, true)+ require.ErrorIs(t, err, context.DeadlineExceeded)++ for _, id := range []string{"container:indexer", "container:torrent"} {+ lease, free, tryErr := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: id})+ require.NoError(t, tryErr)+ require.True(t, free, id)+ lease.Release(t.Context())+ }+}diff --git a/backend/internal/updater/children/execution/service.go b/backend/internal/updater/children/execution/service.goindex 45f40f524..db810a995 100644--- a/backend/internal/updater/children/execution/service.go+++ b/backend/internal/updater/children/execution/service.go@@ -31,7 +31,7 @@ type (Dependencies struct {Config *config.ConfigRoles *role.RoleService- AcquireUpdate func(ctx context.Context) (context.Context, func(), error)+ AcquireUpdate func(ctx context.Context, ids []string) (context.Context, func(), error)UpdateBusy errorRunUpdate func(ctx context.Context, containerID, activityID string) (*updater.Result, error)// RecordUpdate saves the result on the activity and summarizes it as the run outcome.@@ -106,7 +106,7 @@ func (s *Service) update(ctx context.Context, t flow.Task) (any, error) {if err := t.Payload(&input); err != nil {return nil, err}- ctx, release, err := s.deps.AcquireUpdate(ctx)+ ctx, release, err := s.deps.AcquireUpdate(ctx, []string{input.ContainerID})if errors.Is(err, s.deps.UpdateBusy) {return nil, actor.ErrJobRejected}diff --git a/backend/internal/updater/children/recovery/service.go b/backend/internal/updater/children/recovery/service.goindex 73bf5b525..12237c469 100644--- a/backend/internal/updater/children/recovery/service.go+++ b/backend/internal/updater/children/recovery/service.go@@ -5,6 +5,7 @@ import ("context""encoding/json/v2""errors"+ "fmt""slices""strings"@@ -33,7 +34,7 @@ type Service struct {images remoteDigesterpendingUpdates func(ctx context.Context) ([]updater.ImageUpdateRecord, error)activityID func(ctx context.Context) string- acquireUpdate func(ctx context.Context) (context.Context, func(), error)+ acquireUpdate func(ctx context.Context, ids []string) (context.Context, func(), error)applyPending func(ctx context.Context, options arcaneupdater.Options) (*arcaneupdater.Result, error)}@@ -61,7 +62,7 @@ func NewService(images remoteDigester,pendingUpdates func(ctx context.Context) ([]updater.ImageUpdateRecord, error),activityID func(ctx context.Context) string,- acquireUpdate func(ctx context.Context) (context.Context, func(), error),+ acquireUpdate func(ctx context.Context, ids []string) (context.Context, func(), error),applyPending func(ctx context.Context, options arcaneupdater.Options) (*arcaneupdater.Result, error),) *Service {return &Service{@@ -439,7 +440,11 @@ func (s *Service) ConfirmTarget(ctx context.Context, target arcaneupdater.Frozen// ReconcilePending resumes only frozen targets whose original container is unchanged.func (s *Service) ReconcilePending(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {- ctx, release, err := s.acquireUpdate(ctx)+ targetIDs, err := s.TargetRefs(run)+ if err != nil {+ return scheduler.Outcome{Status: scheduler.Waiting}, err+ }+ ctx, release, err := s.acquireUpdate(ctx, targetIDs)if err != nil {return scheduler.Outcome{Status: scheduler.Waiting}, err}@@ -573,6 +578,21 @@ func (s *Service) Started(run scheduler.Run) bool {})}+func (s *Service) TargetRefs(run scheduler.Run) ([]string, error) {+ plan, _, err := frozenPlan(run)+ if err != nil {+ return nil, fmt.Errorf("decode frozen update plan: %w", err)+ }+ ids := make([]string, 0, len(plan.Targets)*3)+ for _, target := range plan.Targets {+ ids = append(ids, target.ContainerID, target.ContainerName)+ if target.ComposeProject != "" {+ ids = append(ids, "compose:"+target.ComposeProject)+ }+ }+ return ids, nil+}+// Unsettled counts the frozen targets a retry would still resume.func (s *Service) Unsettled(run scheduler.Run) (int, error) {plan, found, err := frozenPlan(run)diff --git a/backend/internal/updater/children/recovery/service_test.go b/backend/internal/updater/children/recovery/service_test.goindex 98bd4a580..cbc15d502 100644--- a/backend/internal/updater/children/recovery/service_test.go+++ b/backend/internal/updater/children/recovery/service_test.go@@ -44,7 +44,9 @@ func newTestService(t *testing.T, server *httptest.Server, pending func(context.unresolvedDigests{},pending,func(context.Context) string { return "" },- func(ctx context.Context) (context.Context, func(), error) { return ctx, func() {}, nil },+ func(ctx context.Context, _ []string) (context.Context, func(), error) {+ return ctx, func() {}, nil+ },nil,)}diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.goindex fc141687d..236517cb4 100644--- a/backend/internal/updater/service.go+++ b/backend/internal/updater/service.go@@ -90,9 +90,6 @@ type UpdaterService struct {execution *execution.Servicerecovery *recovery.ServiceautoUpdate *flow.Workflow- // updateMu serializes container updates: compose's recreate pipeline is not safe for- // sibling containers sharing a namespace. Upgrade to per-project if throughput matters.- updateMu sync.Mutex}type updaterDependencies struct {@@ -194,14 +191,18 @@ func NewUpdaterService(imageSvc,service.PendingImageUpdates,activityIDFromContext,- service.acquireUpdate,+ func(ctx context.Context, ids []string) (context.Context, func(), error) {+ return service.acquireUpdate(ctx, ids, false)+ },service.ApplyPending,)// keep: execution callbacks bind to the allocated service and its recoveryservice.execution = execution.NewService(execution.Dependencies{- Config: cfg,- Roles: roles,- AcquireUpdate: service.acquireUpdate,+ Config: cfg,+ Roles: roles,+ AcquireUpdate: func(ctx context.Context, ids []string) (context.Context, func(), error) {+ return service.acquireUpdate(ctx, ids, false)+ },UpdateBusy: errUpdateBusy,RunUpdate: service.runSingleContainerUpdate,RecordUpdate: service.recordSingleContainerUpdate,@@ -242,8 +243,12 @@ func NewUpdaterService(// ApplyPending executes pending image updates. A project or image scope limits the engine's// pending records to its containers; a container scope or a forced run updates each container.func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater.Options) (out *arcaneupdater.Result, err error) {+ containerIDs, err := s.pendingContainerIDs(ctx, options)+ if err != nil {+ return nil, err+ }var release func()- ctx, release, err = s.acquireUpdate(ctx)+ ctx, release, err = s.acquireUpdate(ctx, containerIDs, true)if err != nil {return nil, err}@@ -370,6 +375,46 @@ func (s *UpdaterService) applyBatch(ctx context.Context, options arcaneupdater.Oreturn nil}+func (s *UpdaterService) pendingContainerIDs(ctx context.Context, options arcaneupdater.Options) ([]string, error) {+ if len(options.ResourceIds) > 0 {+ containerIDs, _, _, _, err := s.scopedContainerIDs(ctx, options)+ return containerIDs, err+ }+ if frozen, ok := s.recovery.FrozenRecords(ctx); ok {+ return s.recordContainerIDs(ctx, frozen)+ }+ var records []imageupdate.ImageUpdateRecord+ if err := s.deps.DB.WithContext(ctx).Select("container_id", "repository", "tag").Where("has_update = ? AND project_id = ?", true, "").Find(&records).Error; err != nil {+ return nil, fmt.Errorf("query pending image updates: %w", err)+ }+ pending := make([]updater.ImageUpdateRecord, 0, len(records))+ for _, record := range records {+ pending = append(pending, updater.ImageUpdateRecord{+ ContainerID: record.ContainerID,+ Repository: record.Repository,+ Tag: record.Tag,+ })+ }+ return s.recordContainerIDs(ctx, pending)+}++func (s *UpdaterService) recordContainerIDs(ctx context.Context, records []updater.ImageUpdateRecord) ([]string, error) {+ containerIDs := make([]string, 0, len(records))+ imageRefs := make([]string, 0, len(records))+ for _, record := range records {+ if record.ContainerID != "" {+ containerIDs = append(containerIDs, record.ContainerID)+ continue+ }+ imageRefs = append(imageRefs, record.ImageRef())+ }+ imageContainerIDs, _, _, _, err := s.scopedContainerIDs(ctx, arcaneupdater.Options{Type: "image", ResourceIds: imageRefs})+ if err != nil {+ return nil, err+ }+ return kit.Unique(kit.TrimNonEmpty(slices.Concat(containerIDs, imageContainerIDs))), nil+}+// scopedContainerIDs resolves the requested containers, projects, or images to container IDs.func (s *UpdaterService) scopedContainerIDs(ctx context.Context, options arcaneupdater.Options) ([]string, map[string]string, map[string]string, map[string]bool, error) {requested := kit.TrimNonEmpty(options.ResourceIds)@@ -645,7 +690,7 @@ func (s *UpdaterService) AcceptSingleContainerUpdate(ctx context.Context, contaifunc (s *UpdaterService) runSingleContainerUpdate(ctx context.Context, containerID, activityID string) (out *arcaneupdater.Result, err error) {var release func()- ctx, release, err = s.acquireUpdate(ctx)+ ctx, release, err = s.acquireUpdate(ctx, []string{containerID}, true)if err != nil {return nil, err}@@ -665,8 +710,6 @@ func (s *UpdaterService) runSingleContainerUpdate(ctx context.Context, container}}- s.updateMu.Lock()- defer s.updateMu.Unlock()if ctxErr := ctx.Err(); ctxErr != nil {return out, ctxErr}@@ -797,6 +840,68 @@ func (s *UpdaterService) PullImage(ctx context.Context, imageRef string, progres// PendingImageUpdates returns pending image update records from Arcane's database.func (s *UpdaterService) PendingImageUpdates(ctx context.Context) ([]updater.ImageUpdateRecord, error) {+ records, err := s.loadPendingImageUpdates(ctx)+ if err != nil {+ return nil, err+ }+ if _, batch := ctx.Value(updateAdmissionKey{}).(*updateLeases); !batch || len(records) == 0 {+ return records, nil+ }+ containers, _, _, _, err := s.deps.Docker.GetAllContainers(ctx)+ if err != nil {+ return nil, fmt.Errorf("list containers: %w", err)+ }+ dockerClient, err := s.deps.Docker.GetClient(ctx)+ if err != nil {+ return nil, fmt.Errorf("connect to docker: %w", err)+ }+ keysFor := updateAdmissionKeys(containers)+ configured := map[string]string{}+ if slices.ContainsFunc(records, func(record updater.ImageUpdateRecord) bool { return record.ContainerID == "" }) {+ for _, summary := range containers {+ configured[summary.ID] = refs.NormalizeImageUpdateRef(imageref.ConfiguredImage(ctx, dockerClient, summary))+ }+ }+ admitted := make([]updater.ImageUpdateRecord, 0, len(records))+ for _, record := range records {+ containerIDs, matchErr := func() ([]string, error) {+ if record.ContainerID != "" {+ return []string{record.ContainerID}, nil+ }+ if record.ImageRef() == "" {+ return nil, nil+ }+ inspected, inspectErr := dockerClient.ImageInspect(ctx, record.ImageRef())+ if inspectErr != nil && !errdefs.IsNotFound(inspectErr) {+ return nil, fmt.Errorf("inspect pending image %s: %w", record.ImageRef(), inspectErr)+ }+ oldIDs := kit.TrimNonEmpty([]string{inspected.ID, record.ID})+ normalized := refs.NormalizeImageUpdateRef(record.ImageRef())+ var matched []string+ for _, summary := range containers {+ if slices.Contains(oldIDs, summary.ImageID) || (normalized != "" && configured[summary.ID] == normalized) {+ matched = append(matched, summary.ID)+ }+ }+ return matched, nil+ }()+ if matchErr != nil {+ return nil, matchErr+ }+ _, _, acquireErr := s.admitUpdate(ctx, keysFor(containerIDs), false)+ if errors.Is(acquireErr, errUpdateBusy) {+ slog.InfoContext(ctx, "skipping pending update for a container another update holds", "imageRef", record.ImageRef(), "containerId", record.ContainerID)+ continue+ }+ if acquireErr != nil {+ return nil, acquireErr+ }+ admitted = append(admitted, record)+ }+ return admitted, nil+}++func (s *UpdaterService) loadPendingImageUpdates(ctx context.Context) ([]updater.ImageUpdateRecord, error) {scoped, isScoped := ctx.Value(scopedPendingKey{}).(map[string]string)outOfScope := func(record updater.ImageUpdateRecord) bool {_, inScope := scoped[record.ContainerID]@@ -1002,10 +1107,33 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.Sif pullErr != nil {return pullErr}+ containers, _, _, _, listErr := s.deps.Docker.GetAllContainers(ctx)+ if listErr != nil {+ return fmt.Errorf("list containers: %w", listErr)+ }+ stopsProcess := target.InstanceType != "agent"+ keys := updateAdmissionKeys(containers)(func() []string {+ if !stopsProcess {+ return []string{target.ContainerID}+ }+ allIDs := make([]string, 0, len(containers))+ for _, summary := range containers {+ allIDs = append(allIDs, summary.ID)+ }+ return allIDs+ }())+ ctx, release, acquireErr := s.admitUpdate(ctx, keys, false)+ if errors.Is(acquireErr, errUpdateBusy) {+ return common.Classify(common.ErrConflict, errors.New("another container update is in progress; the self-update stays pending for the next run"))+ }+ if acquireErr != nil {+ return acquireErr+ }+ defer release()// A server self-update stops this process before the run completes, so the// activity is flagged first and startup reconciliation finalizes it.- if activityID := activityIDFromContext(ctx); target.InstanceType != "agent" && activityID != "" {+ if activityID := activityIDFromContext(ctx); stopsProcess && activityID != "" {s.appendAutoUpdateActivityMessage(ctx,activityID,@@ -1018,16 +1146,49 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.S}}- if _, err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, updater.SelfUpdateTarget{ //exhaustruct:enforce+ upgraderID, err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, updater.SelfUpdateTarget{ //exhaustruct:enforceContainerID: target.ContainerID,ContainerName: target.ContainerName,InstanceType: target.InstanceType,Labels: target.Labels,NewImageRef: target.NewImageRef,PullImageRef: pullImageRef,- }); err != nil {+ })+ if err != nil {return fmt.Errorf("CLI upgrade failed: %w", err)}+ done := make(chan struct{})+ go func() {+ defer close(done)+ waitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), timeouts.DefaultDeployWait)+ defer cancel()+ if stopsProcess || upgraderID == "" {+ <-waitCtx.Done()+ return+ }+ dockerClient, clientErr := s.deps.Docker.GetClient(waitCtx)+ if clientErr != nil {+ slog.WarnContext(waitCtx, "cannot watch agent upgrader; holding its update keys until the deploy wait ends", "upgraderId", upgraderID, "error", clientErr)+ <-waitCtx.Done()+ return+ }+ wait := dockerClient.ContainerWait(waitCtx, upgraderID, client.ContainerWaitOptions{Condition: container.WaitConditionNotRunning})+ select {+ case <-wait.Result:+ case waitErr := <-wait.Error:+ if !errdefs.IsNotFound(waitErr) {+ slog.WarnContext(waitCtx, "agent upgrader wait failed; holding its update keys until the deploy wait ends", "upgraderId", upgraderID, "error", waitErr)+ <-waitCtx.Done()+ }+ }+ }()+ if leases, ok := ctx.Value(updateAdmissionKey{}).(*updateLeases); ok {+ leases.mu.Lock()+ for _, key := range kit.Ternary(stopsProcess, leases.keys, keys) {+ leases.retain[key] = done+ }+ leases.mu.Unlock()+ }if progress, ok := ctx.Value(updateProgressKey{}).(*updateProgress); ok {progress.mu.Lock()progress.selfTriggered = true@@ -1447,20 +1608,6 @@ func addNormalizedImageUpdateRef(ctx context.Context, out map[string]struct{}, islog.DebugContext(ctx, "skipping invalid image reference", append(attrs, "source", source, "imageRef", imageRef)...)}-func (s *UpdaterService) acquireUpdate(ctx context.Context) (context.Context, func(), error) {- if ctx.Value(updateAdmissionKey{}) == s {- return ctx, func() {}, nil- }- lease, admitted, err := s.admission.TryAcquire(ctx, scheduler.AdmissionKey{Scope: "updater"})- if err != nil {- return ctx, nil, err- }- if !admitted {- return ctx, nil, errUpdateBusy- }- return context.WithValue(ctx, updateAdmissionKey{}, s), func() { lease.Release(ctx) }, nil-}-func (s *UpdaterService) PruneHistory(ctx context.Context, retentionDays int) (int64, error) {if retentionDays <= 0 {return 0, nil@@ -1873,7 +2020,11 @@ func (s *UpdaterService) applyAutoUpdate(ctx context.Context, t flow.Task) (any,if !ok {return nil, errors.New("auto-update requires a job run")}- ctx, release, err := s.acquireUpdate(ctx)+ targetIDs, err := s.recovery.TargetRefs(run)+ if err != nil {+ return nil, common.Classify(common.ErrUnavailable, err)+ }+ ctx, release, err := s.acquireUpdate(ctx, targetIDs, false)if errors.Is(err, errUpdateBusy) {// A plan whose every target already settled needs no updater, so it finishes normally.if unsettled, unsettledErr := s.recovery.Unsettled(run); unsettledErr == nil && unsettled == 0 {diff --git a/backend/internal/updater/service_test.go b/backend/internal/updater/service_test.goindex 2aa675d0f..5724f4d1f 100644--- a/backend/internal/updater/service_test.go+++ b/backend/internal/updater/service_test.go@@ -332,10 +332,18 @@ func TestUpdaterService_ResultFromModulePreservesRestarted(t *testing.T) {func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {ctx := t.Context()+ newAdmission := func(t *testing.T) *runs.Admission {+ t.Helper()+ harness := flowtest.New(t, activity.NewActivityService(setupProjectTestDB(t), nil))+ admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())+ require.NoError(t, admission.Register(harness.Runtime))+ harness.Start(t)+ return admission+ }t.Run("server label triggers upgrade with system user", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)require.NoError(t, svcErr)err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{@@ -356,7 +364,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("engine target keeps the selected tag and pulls the frozen digest", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)require.NoError(t, svcErr)frozen := svc.recovery.WithSingleTarget(ctx, &arcaneupdater.FrozenUpdateTarget{@@ -379,7 +387,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("agent label triggers upgrade", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)require.NoError(t, svcErr)err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane-agent", map[string]string{@@ -392,7 +400,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("non Arcane labels fail without triggering upgrade", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)require.NoError(t, svcErr)err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "demo", map[string]string{@@ -406,7 +414,7 @@ func TestUpdaterService_TriggerSelfUpdateViaCLI(t *testing.T) {t.Run("upgrade errors are wrapped", func(t *testing.T) {mockUpgrade := &mockSystemUpgradeService{triggerError: errors.New("upgrade failed")}- svc, svcErr := NewUpdaterService(nil, nil, nil, nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, nil, nil)+ svc, svcErr := NewUpdaterService(nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, mockUpgrade, nil, nil, newAdmission(t), nil)require.NoError(t, svcErr)err := svc.TriggerSelfUpdateViaCLI(ctx, "test", "container-1", "arcane", map[string]string{@@ -863,13 +871,13 @@ func TestUpdaterService_AcceptSingleContainerUpdateUsesQueuedActivity(t *testingharness := flowtest.New(t, activityService)admission := runs.NewAdmission(harness.Runtime.Service(), t.Name())require.NoError(t, admission.Register(harness.Runtime))- svc, err := NewUpdaterService(db, nil, testDockerService, nil, nil, nil, nil, nil, nil, nil, activityService, nil, admission, nil)+ svc, err := NewUpdaterService(db, nil, newEmptyContainerListDockerService(t), nil, nil, nil, nil, nil, nil, nil, activityService, nil, admission, nil)require.NoError(t, err)require.NoError(t, svc.execution.RegisterWorkflows(harness.Engine))harness.Start(t)// Holding the update admission makes the workflow hand its task back until the lease is released.- held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater"})+ held, admitted, err := admission.TryAcquire(t.Context(), scheduler.AdmissionKey{Scope: "updater", ID: "id:missing-container"})require.NoError(t, err)require.True(t, admitted)var releaseOnce sync.Once@@ -1293,7 +1301,7 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgrade(svc, svcErr := NewUpdaterService(db,nil,- nil,+ (&docker.DockerClientService{}).WithClient(newTestDockerClient(t, server)),nil,imageupdate.NewImageUpdateService(db, nil, nil, newEmptyContainerListDockerService(t), nil, nil, nil),nil,diff --git a/backend/pkg/scheduler/runs/actors_test.go b/backend/pkg/scheduler/runs/actors_test.goindex 4009abb70..5bfaf8639 100644--- a/backend/pkg/scheduler/runs/actors_test.go+++ b/backend/pkg/scheduler/runs/actors_test.go@@ -319,6 +319,44 @@ func TestAdmissionAndCompletionRejectStaleTokens(t *testing.T) {require.Equal(t, "new-token", persisted.Owner)}+func TestAdmissionAcquireWaitsForRelease(t *testing.T) {+ host := francistest.New(t)+ admission := NewAdmission(host.Service(), "epoch")+ require.NoError(t, admission.Register(host))+ francistest.Start(t, host)+ key := st.AdmissionKey{Scope: "updater", ID: "container:solo"}+ held, admitted, err := admission.TryAcquire(t.Context(), key)+ require.NoError(t, err)+ require.True(t, admitted)++ canceledCtx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond)+ defer cancel()+ _, err = admission.Acquire(canceledCtx, key)+ require.ErrorIs(t, err, context.DeadlineExceeded)++ acquired := make(chan *Lease, 1)+ go func() {+ lease, acquireErr := admission.Acquire(t.Context(), key)+ if acquireErr != nil {+ lease = nil+ }+ acquired <- lease+ }()+ select {+ case <-acquired:+ require.FailNow(t, "acquire returned while the key was held")+ case <-time.After(400 * time.Millisecond):+ }+ held.Release(t.Context())+ select {+ case lease := <-acquired:+ require.NotNil(t, lease)+ lease.Release(t.Context())+ case <-time.After(5 * time.Second):+ require.FailNow(t, "acquire was not admitted after release")+ }+}+func TestLostDispatchAcknowledgementRetainsOneExecution(t *testing.T) {q := newResolutionQueue(t)entered := make(chan struct{})diff --git a/backend/pkg/scheduler/runs/admission.go b/backend/pkg/scheduler/runs/admission.goindex 73a2d9158..64e50c75b 100644--- a/backend/pkg/scheduler/runs/admission.go+++ b/backend/pkg/scheduler/runs/admission.go@@ -72,6 +72,22 @@ func (a *Admission) TryAcquire(ctx context.Context, key st.AdmissionKey) (*Leasereturn &Lease{admission: a, id: id, token: token}, true, nil}+func (a *Admission) Acquire(ctx context.Context, key st.AdmissionKey) (*Lease, error) {+ ticker := time.NewTicker(250 * time.Millisecond)+ defer ticker.Stop()+ for {+ lease, admitted, err := a.TryAcquire(ctx, key)+ if err != nil || admitted {+ return lease, err+ }+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-ticker.C:+ }+ }+}+func (l *Lease) Release(ctx context.Context) {if l == nil {return