Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
perf(imageupdate): dedupe and parallelize container tag checks (#4109)
Kyle Mendell57812f8f0511
diff --git a/backend/internal/imageupdate/service_test.go b/backend/internal/imageupdate/service_test.goindex e1be01a45..7ebbd7f18 100644--- a/backend/internal/imageupdate/service_test.go+++ b/backend/internal/imageupdate/service_test.go@@ -85,6 +85,11 @@ func setupImageUpdateRegistryTestDBInternal(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)+ // Every pooled connection to a private in-memory database sees its own+ // empty schema, so concurrent scan workers must share a single connection.+ sqlDB, err := db.DB()+ require.NoError(t, err)+ sqlDB.SetMaxOpenConns(1)require.NoError(t, db.AutoMigrate(®istry.ContainerRegistry{},&kv.KVEntry{},@@ -2415,11 +2420,13 @@ func (testProjectRow) TableName() string { return "projects" }func TestContainerTagChecksPersistIndependentPoliciesInternal(t *testing.T) {db := setupImageUpdateRegistryTestDBInternal(t)+ var tagListings atomic.Int64registryServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {if !strings.HasSuffix(r.URL.Path, "/tags/list") {http.NotFound(w, r)return}+ tagListings.Add(1)w.Header().Set("Content-Type", "application/json")require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"name": "team/app", "tags": []string{"1.0.0", "1.1.0", "2.0.0"}}))}))@@ -2462,6 +2469,7 @@ func TestContainerTagChecksPersistIndependentPoliciesInternal(t *testing.T) {require.Len(t, checks, 2)require.Equal(t, "1.1.0", checks["one"].LatestVersion)require.Equal(t, "2.0.0", checks["two"].LatestVersion)+ require.EqualValues(t, 1, tagListings.Load(), "replicas of one repository share a single tag listing per scan")var records []ImageUpdateRecordrequire.NoError(t, db.Order("container_id").Find(&records).Error)require.Len(t, records, 2)@@ -2588,3 +2596,68 @@ func TestContainerTagChecksUseRegistryTagTimeoutInternal(t *testing.T) {})}}++func TestContainerTagChecksPersistResultsFinishedBeforeScanDeadlineInternal(t *testing.T) {+ db := setupImageUpdateRegistryTestDBInternal(t)+ t.Setenv("REGISTRY_TAG_TIMEOUT", "30")+ settingsService := newImageUpdateTestSettingsServiceInternal(t, "30", "30")+ registryServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case strings.HasSuffix(r.URL.Path, "/team/fast/tags/list"):+ w.Header().Set("Content-Type", "application/json")+ require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"name": "team/fast", "tags": []string{"1.0.0", "1.1.0"}}))+ case strings.HasSuffix(r.URL.Path, "/team/slow/tags/list"):+ // Stall until the scan deadline cancels the request.+ <-r.Context().Done()+ default:+ http.NotFound(w, r)+ }+ }))+ defer registryServer.Close()+ registryURL, err := url.Parse(registryServer.URL)+ require.NoError(t, err)+ imageRefs := map[string]string{"fast": registryURL.Host + "/team/fast:1.0.0", "slow": registryURL.Host + "/team/slow:1.0.0"}+ imageIDs := map[string]string{"fast": digest.FromString("fast").String(), "slow": digest.FromString("slow").String()}+ nameFor := func(path string) string {+ if strings.Contains(path, "slow") {+ return "slow"+ }+ return "fast"+ }+ dockerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ name := nameFor(r.URL.Path)+ switch {+ case strings.HasSuffix(r.URL.Path, "/containers/json"):+ require.NoError(t, json.NewEncoder(w).Encode([]dockertypescontainer.Summary{+ {ID: "fast", Image: imageRefs["fast"], ImageID: imageIDs["fast"]},+ {ID: "slow", Image: imageRefs["slow"], ImageID: imageIDs["slow"]},+ }))+ case strings.Contains(r.URL.Path, "/images/"):+ require.NoError(t, json.NewEncoder(w).Encode(dockertypesimage.InspectResponse{ID: imageIDs[name], RepoTags: []string{imageRefs[name]}}))+ case strings.Contains(r.URL.Path, "/containers/"):+ require.NoError(t, json.NewEncoder(w).Encode(dockertypescontainer.InspectResponse{ID: name, Image: imageIDs[name], Config: &dockertypescontainer.Config{Image: imageRefs[name]}}))+ default:+ http.NotFound(w, r)+ }+ }))+ defer dockerServer.Close()+ registryService := registry.NewContainerRegistryService(db, func(context.Context) (registry.RegistryDaemonClient, error) {+ return &fakeRegistryDaemonClient{distributionInspectFn: func(context.Context, string, client.DistributionInspectOptions) (client.DistributionInspectResult, error) {+ return client.DistributionInspectResult{}, errors.New("manifest not found")+ }}, nil+ }, nil, registryServer.Client()).WithSettingsService(settingsService)+ svc := NewImageUpdateService(db, settingsService, registryService, &docker.DockerClientService{Client: newImageUpdateTestDockerClientInternal(t, dockerServer)}, nil, nil, nil)++ scanCtx, cancel := context.WithTimeout(t.Context(), time.Second)+ defer cancel()+ _, err = svc.checkContainerTagUpdatesInternal(scanCtx, []string{imageRefs["fast"], imageRefs["slow"]}, nil)+ require.ErrorContains(t, err, context.DeadlineExceeded.Error())+ var records []ImageUpdateRecord+ require.NoError(t, db.Find(&records).Error)+ require.Len(t, records, 1, "only the check that finished before the scan deadline is stored")+ require.Equal(t, "container::fast", records[0].ID)+ require.True(t, records[0].HasUpdate)+ require.Equal(t, "1.1.0", mo.PointerToOption(records[0].LatestVersion).OrEmpty())+ require.Empty(t, mo.PointerToOption(records[0].LastError).OrEmpty(), "a stored result must not be overwritten by the cancellation")+}diff --git a/backend/internal/imageupdate/tag_updates.go b/backend/internal/imageupdate/tag_updates.goindex d282acb3b..9412310de 100644--- a/backend/internal/imageupdate/tag_updates.go+++ b/backend/internal/imageupdate/tag_updates.go@@ -4,7 +4,9 @@ import ("context""fmt""log/slog"+ "slices""strings"+ "sync""time""emperror.dev/errors"@@ -12,6 +14,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/registry""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/imageref""github.com/getarcaneapp/arcane/types/v2/containerregistry"imageupdatetypes "github.com/getarcaneapp/arcane/types/v2/imageupdate"@@ -21,42 +24,128 @@ import ("go.getarcane.app/updater/pkg/utils/tagpolicy""go.getarcane.app/updater/refs"+ "golang.org/x/sync/errgroup"+ "golang.org/x/sync/singleflight""gorm.io/gorm")+// tagCheckWorkers bounds concurrent container policy evaluations per scan; the+// registry limiter still caps concurrent requests per registry host.+const tagCheckWorkers = 10++// staleDigestRecordDeleteChunk bounds the IN list used to clear container+// records whose policy switched to digest tracking.+const staleDigestRecordDeleteChunk = 500++// tagRegistryInternal adapts the registry service for one container tag scan.+// Successful tag listings are shared per repository and digest lookups per+// reference so replicas of the same image cost one registry request, and the+// scan's manager-provided credentials never leak into the service caches.type tagRegistryInternal struct {service *registry.ContainerRegistryServicecredentials []containerregistry.Credentialdocker *docker.DockerClientServicesettings *settings.SettingsService+ tags *scanRegistryMemoInternal[[]string]+ digests *scanRegistryMemoInternal[string]+}++func newTagRegistryInternal(service *registry.ContainerRegistryService, credentials []containerregistry.Credential, dockerService *docker.DockerClientService, settingsService *settings.SettingsService) tagRegistryInternal {+ return tagRegistryInternal{+ service: service,+ credentials: credentials,+ docker: dockerService,+ settings: settingsService,+ tags: newScanRegistryMemoInternal[[]string](),+ digests: newScanRegistryMemoInternal[string](),+ }+}++// scanRegistryMemoInternal memoizes successful registry lookups for one scan+// and coalesces concurrent misses for the same key into a single request.+// Failures are not stored: each waiting caller receives the shared error and+// the next caller retries, so a transient fault never sticks for the scan.+type scanRegistryMemoInternal[V any] struct {+ flight singleflight.Group+ mu sync.Mutex+ values map[string]V+}++func newScanRegistryMemoInternal[V any]() *scanRegistryMemoInternal[V] {+ return &scanRegistryMemoInternal[V]{values: map[string]V{}}+}++func (m *scanRegistryMemoInternal[V]) doInternal(key string, fetch func() (V, error)) (V, error) {+ m.mu.Lock()+ value, ok := m.values[key]+ m.mu.Unlock()+ if ok {+ return value, nil+ }+ shared, err, _ := m.flight.Do(key, func() (any, error) {+ fetched, fetchErr := fetch()+ if fetchErr != nil {+ return nil, fetchErr+ }+ m.mu.Lock()+ m.values[key] = fetched+ m.mu.Unlock()+ return fetched, nil+ })+ var zero V+ if err != nil {+ return zero, err+ }+ value, ok = shared.(V)+ if !ok {+ return zero, errors.New("registry lookup returned an unexpected result type")+ }+ return value, nil}func (r tagRegistryInternal) ListTags(ctx context.Context, imageRef string) ([]string, error) {if r.service == nil {return nil, errors.New("registry service unavailable")}- return r.service.ListImageTags(ctx, imageRef, r.credentials)+ fetch := func() ([]string, error) { return r.service.ListImageTags(ctx, imageRef, r.credentials) }+ parsed, err := refs.NormalizeReference(imageRef)+ if err != nil || r.tags == nil {+ return fetch()+ }+ tags, err := r.tags.doInternal(parsed.RegistryHost+"/"+parsed.Repository, fetch)+ if err != nil {+ return nil, err+ }+ // Callers may sort or filter the listing in place.+ return slices.Clone(tags), nil}func (r tagRegistryInternal) ImageDigest(ctx context.Context, imageRef string) (string, error) {if r.service == nil {return "", errors.New("registry service unavailable")}- timeoutSeconds := 0- if r.settings != nil {- timeoutSeconds = r.settings.GetSettingsConfig().RegistryTimeout.AsInt()- }- digestCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(timeoutSeconds, timeouts.DefaultRegistry))- defer cancel()- result, err := r.service.InspectImageDigest(digestCtx, imageRef, r.credentials)- if err != nil {- return "", err+ fetch := func() (string, error) {+ timeoutSeconds := 0+ if r.settings != nil {+ timeoutSeconds = r.settings.GetSettingsConfig().RegistryTimeout.AsInt()+ }+ digestCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(timeoutSeconds, timeouts.DefaultRegistry))+ defer cancel()+ result, err := r.service.InspectImageDigest(digestCtx, imageRef, r.credentials)+ if err != nil {+ return "", err+ }+ if result == nil {+ return "", errors.New("registry returned no digest")+ }+ return result.Digest, nil}- if result == nil {- return "", errors.New("registry returned no digest")+ parsed, err := refs.NormalizeReference(imageRef)+ if err != nil || r.digests == nil {+ return fetch()}- return result.Digest, nil+ return r.digests.doInternal(parsed.NormalizedRef, fetch)}func (s *ImageUpdateService) checkContainerTagUpdatesInternal(ctx context.Context, imageRefs []string, credentials []containerregistry.Credential) (map[string]*imageupdatetypes.Response, error) {@@ -81,7 +170,7 @@ func (s *ImageUpdateService) checkContainerTagUpdatesInternal(ctx context.Contexreturn results, errors.WrapIf(err, "list containers for tag update checks")}- adapter := tagRegistryInternal{service: s.registryService, credentials: credentials, docker: s.dockerService, settings: s.settingsService}+ adapter := newTagRegistryInternal(s.registryService, credentials, s.dockerService, s.settingsService)engine, err := updater.New(updater.Config{RegistryTagLister: adapter, RegistryDigestResolver: adapter, DockerClientProvider: adapter, Settings: adapter})if err != nil {return results, err@@ -92,26 +181,64 @@ func (s *ImageUpdateService) checkContainerTagUpdatesInternal(ctx context.Contex}}()policy := updater.DefaultLabelPolicy()+ candidates := make([]container.Summary, 0, len(listed.Items))+ var staleDigestContainerIDs []stringfor _, cnt := range listed.Items {if !wanted[refs.NormalizeImageUpdateRef(cnt.Image)] {continue}tagPolicy, policyErr := tagpolicy.Resolve(cnt.Image, policy.TagPolicy(cnt.Labels))if policyErr == nil && tagPolicy.Strategy == "digest" {- if s.db != nil {- if err := s.db.WithContext(ctx).Where("container_id = ?", cnt.ID).Delete(&ImageUpdateRecord{}).Error; err != nil {- return results, err- }- }+ staleDigestContainerIDs = append(staleDigestContainerIDs, cnt.ID)continue}- result := s.checkContainerTagInternal(ctx, engine, cnt)- results[cnt.ID] = result- if err := s.saveContainerTagResultInternal(ctx, cnt, result); err != nil {- return results, err+ candidates = append(candidates, cnt)+ }+ if err := s.deleteContainerUpdateRecordsInternal(ctx, staleDigestContainerIDs); err != nil {+ return results, err+ }++ // Policies are evaluated concurrently; the shared adapter dedupes registry+ // requests across replicas and the registry limiter caps them per host.+ // Each result is persisted as soon as its check completes, so a scan that+ // reaches its deadline keeps every check that finished before it. Saves are+ // serialized and run on the group context: once the scan is cancelled the+ // remaining writes fail instead of replacing stored results with+ // cancellation errors.+ checks := make([]*imageupdatetypes.Response, len(candidates))+ var saveMu sync.Mutex+ g, groupCtx := errgroup.WithContext(ctx)+ g.SetLimit(tagCheckWorkers)+ for i, cnt := range candidates {+ g.Go(func() (workerErr error) {+ defer utils.RecoverToError(&workerErr, "container tag check worker", "containerId", cnt.ID)+ checks[i] = s.checkContainerTagInternal(groupCtx, engine, cnt)+ saveMu.Lock()+ defer saveMu.Unlock()+ return s.saveContainerTagResultInternal(groupCtx, cnt, checks[i])+ })+ }+ waitErr := g.Wait()+ for i, cnt := range candidates {+ if checks[i] != nil {+ results[cnt.ID] = checks[i]+ }+ }+ return results, waitErr+}++// deleteContainerUpdateRecordsInternal clears the stored results of containers+// whose policy no longer tracks tags, in bounded IN-list chunks.+func (s *ImageUpdateService) deleteContainerUpdateRecordsInternal(ctx context.Context, containerIDs []string) error {+ if s.db == nil {+ return nil+ }+ for chunk := range slices.Chunk(containerIDs, staleDigestRecordDeleteChunk) {+ if err := s.db.WithContext(ctx).Where("container_id IN ?", chunk).Delete(&ImageUpdateRecord{}).Error; err != nil {+ return err}}- return results, nil+ return nil}func (s *ImageUpdateService) checkContainerTagInternal(ctx context.Context, engine *updater.Service, cnt container.Summary) *imageupdatetypes.Response {