Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: retry rate limited update checks (#2639)
Kyle Mendell8671ba4b2431
diff --git a/backend/internal/services/container_registry_service.go b/backend/internal/services/container_registry_service.goindex 2cc0afc7d..00aa105ab 100644--- a/backend/internal/services/container_registry_service.go+++ b/backend/internal/services/container_registry_service.go@@ -11,6 +11,7 @@ import ("sync""time"+ backoff "github.com/cenkalti/backoff/v5""golang.org/x/sync/singleflight"cerrdefs "github.com/containerd/errdefs"@@ -733,28 +734,73 @@ func (s *ContainerRegistryService) inspectImageDigestInternal(ctx context.Contexreturn nil, err}- result, err := s.inspectImageDigestViaDaemonInternal(ctx, parts.NormalizedRef, parts.RegistryHost, externalCreds)- if err == nil {- return result, nil- }- if !isDistributionFallbackEligibleInternal(err) {- return result, err- }+ var lastResult *registryDigestResult+ var lastErr error+ fallbackWarningLogged := false++ bo := backoff.NewExponentialBackOff()+ bo.InitialInterval = 500 * time.Millisecond+ bo.MaxInterval = 10 * time.Second+ bo.RandomizationFactor = 0.3++ _, retryErr := backoff.Retry(ctx, func() (*registryDigestResult, error) {+ result, err := s.inspectImageDigestViaDaemonInternal(ctx, parts.NormalizedRef, parts.RegistryHost, externalCreds)+ if err == nil {+ lastResult = result+ return result, nil+ }++ if isRateLimitErrorInternal(err) {+ lastResult = result+ lastErr = err+ slog.DebugContext(ctx, "rate limited by registry, will retry",+ "registry", parts.RegistryHost,+ "imageRef", parts.NormalizedRef)+ return nil, err+ }++ if !isDistributionFallbackEligibleInternal(err) {+ lastResult = result+ lastErr = err+ return nil, backoff.Permanent(err)+ }++ if !fallbackWarningLogged {+ fallbackWarningLogged = true+ slog.WarnContext(ctx, "distribution inspect unavailable, falling back to direct registry digest lookup",+ "imageRef", parts.NormalizedRef,+ "registry", parts.RegistryHost,+ "error", err.Error())+ }++ fallbackResult, fallbackErr := s.inspectImageDigestViaRegistryInternal(ctx, parts.RegistryHost, parts.Repository, parts.Tag, externalCreds)+ if fallbackErr == nil {+ lastResult = fallbackResult+ return fallbackResult, nil+ }++ if isRateLimitErrorInternal(fallbackErr) {+ lastResult = fallbackResult+ lastErr = fallbackErr+ slog.DebugContext(ctx, "rate limited by registry on fallback, will retry",+ "registry", parts.RegistryHost,+ "imageRef", parts.NormalizedRef)+ return nil, fallbackErr+ }- slog.WarnContext(ctx, "distribution inspect unavailable, falling back to direct registry digest lookup",- "imageRef", parts.NormalizedRef,- "registry", parts.RegistryHost,- "error", err.Error())+ lastResult = fallbackResult+ lastErr = fmt.Errorf("daemon digest lookup failed; registry fallback failed: %w", errors.Join(err, fallbackErr))+ return nil, backoff.Permanent(lastErr)+ }, backoff.WithBackOff(bo), backoff.WithMaxTries(5))- fallbackResult, fallbackErr := s.inspectImageDigestViaRegistryInternal(ctx, parts.RegistryHost, parts.Repository, parts.Tag, externalCreds)- if fallbackErr == nil {- return fallbackResult, nil+ if retryErr != nil {+ if errors.Is(retryErr, context.Canceled) || errors.Is(retryErr, context.DeadlineExceeded) {+ return lastResult, retryErr+ }+ return lastResult, lastErr}- return fallbackResult, fmt.Errorf(- "daemon digest lookup failed; registry fallback failed: %w",- errors.Join(err, fallbackErr),- )+ return lastResult, nil}func (s *ContainerRegistryService) inspectImageDigestViaDaemonInternal(ctx context.Context, normalizedRef, registryHost string, externalCreds []containerregistry.Credential) (*registryDigestResult, error) {@@ -1220,8 +1266,6 @@ func isUnauthorizedRegistryErrorInternal(err error) bool {"no basic auth credentials","access denied","incorrect username or password",- "toomanyrequests",- "unauthenticated pull rate limit","status: 401","status 401","status: 403",@@ -1237,6 +1281,27 @@ func isUnauthorizedRegistryErrorInternal(err error) bool {return false}+func isRateLimitErrorInternal(err error) bool {+ if err == nil {+ return false+ }+ errLower := strings.ToLower(err.Error())+ indicators := []string{+ "toomanyrequests",+ "rate limit",+ "too many requests",+ "status: 429",+ "status 429",+ "retry-after",+ }+ for _, indicator := range indicators {+ if strings.Contains(errLower, indicator) {+ return true+ }+ }+ return false+}+func isDockerHubRegistryInternal(registryHost string) bool {return utilsregistry.NormalizeRegistryForComparison(registryHost) == "docker.io"}diff --git a/backend/internal/services/image_update_service.go b/backend/internal/services/image_update_service.goindex 5400af235..74a199683 100644--- a/backend/internal/services/image_update_service.go+++ b/backend/internal/services/image_update_service.go@@ -13,6 +13,7 @@ import ("github.com/getarcaneapp/arcane/backend/internal/models""github.com/getarcaneapp/arcane/backend/pkg/libarcane/crypto"imageupdatecore "github.com/getarcaneapp/arcane/backend/pkg/libarcane/imageupdate"+ "github.com/getarcaneapp/arcane/backend/pkg/libarcane/ratelimit"registry "github.com/getarcaneapp/arcane/backend/pkg/libarcane/registryauth""github.com/getarcaneapp/arcane/types/containerregistry""github.com/getarcaneapp/arcane/types/imageupdate"@@ -30,6 +31,7 @@ type ImageUpdateService struct {dockerService *DockerClientServiceeventService *EventServicenotificationService *NotificationService+ registryLimiter *ratelimit.RegistryRateLimiter}type ImageParts struct {@@ -54,6 +56,7 @@ func NewImageUpdateService(db *database.DB, settingsService *SettingsService, redockerService: dockerService,eventService: eventService,notificationService: notificationService,+ registryLimiter: ratelimit.NewRegistryRateLimiter(),}}@@ -904,6 +907,17 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefsfor _, img := range images {g.Go(func() error {+ registry := img.parts.Registry++ if err := s.registryLimiter.Acquire(groupCtx, registry); err != nil {+ slog.DebugContext(groupCtx, "skipping image check: registry limiter acquire failed",+ "imageRef", img.canonicalRef,+ "registry", registry,+ "error", err.Error())+ return err+ }+ defer s.registryLimiter.Release(registry)+res, snapshot := s.checkSingleImageInBatchInternal(groupCtx, resolvedCreds, img.parts)mu.Lock()@@ -921,6 +935,7 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefsif err := g.Wait(); err != nil {slog.ErrorContext(ctx, "Batch check error", "error", err)+ return results, err}successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)diff --git a/backend/pkg/libarcane/ratelimit/registry_rate_limiter.go b/backend/pkg/libarcane/ratelimit/registry_rate_limiter.gonew file mode 100644index 000000000..1a01bd9e1--- /dev/null+++ b/backend/pkg/libarcane/ratelimit/registry_rate_limiter.go@@ -0,0 +1,89 @@+package ratelimit++import (+ "context"+ "strings"+ "sync"++ "golang.org/x/sync/semaphore"+)++type RegistryRateLimiter struct {+ mu sync.RWMutex+ semaphores map[string]*semaphore.Weighted+ limits map[string]int64+ defaultLimit int64+}++var defaultRegistryLimits = map[string]int64{+ "ghcr.io": 3,+ "docker.io": 5,+ "quay.io": 4,+ "gcr.io": 4,+}++const defaultRegistryConcurrencyLimit int64 = 3++func NewRegistryRateLimiter() *RegistryRateLimiter {+ return &RegistryRateLimiter{+ semaphores: make(map[string]*semaphore.Weighted),+ limits: defaultRegistryLimits,+ defaultLimit: defaultRegistryConcurrencyLimit,+ }+}++func (r *RegistryRateLimiter) Acquire(ctx context.Context, registry string) error {+ sem := r.getSemaphoreInternal(registry)+ return sem.Acquire(ctx, 1)+}++func (r *RegistryRateLimiter) Release(registry string) {+ normalized := normalizeRegistryKeyInternal(registry)+ r.mu.RLock()+ sem, ok := r.semaphores[normalized]+ r.mu.RUnlock()+ if ok {+ sem.Release(1)+ }+}++func (r *RegistryRateLimiter) getSemaphoreInternal(registry string) *semaphore.Weighted {+ normalized := normalizeRegistryKeyInternal(registry)++ r.mu.RLock()+ sem, ok := r.semaphores[normalized]+ r.mu.RUnlock()+ if ok {+ return sem+ }++ r.mu.Lock()+ defer r.mu.Unlock()++ if sem, ok = r.semaphores[normalized]; ok {+ return sem+ }++ limit := r.limitForRegistryInternal(normalized)+ sem = semaphore.NewWeighted(limit)+ r.semaphores[normalized] = sem+ return sem+}++func (r *RegistryRateLimiter) limitForRegistryInternal(registry string) int64 {+ if limit, ok := r.limits[registry]; ok {+ return limit+ }+ if strings.Contains(registry, ".ecr.") && strings.Contains(registry, ".amazonaws.com") {+ return 4+ }+ return r.defaultLimit+}++func normalizeRegistryKeyInternal(registry string) string {+ normalized := strings.ToLower(strings.TrimSpace(registry))+ if normalized == "registry-1.docker.io" || normalized == "index.docker.io" {+ return "docker.io"+ }+ return normalized+}