Skip to content

Modern Docker Management, Designed for Everyone

fix: retry rate limited update checks (#2639)

Kyle Mendell8671ba4b2431
diff --git a/backend/internal/services/container_registry_service.go b/backend/internal/services/container_registry_service.go
index 2cc0afc7d..00aa105ab 100644
--- a/backend/internal/services/container_registry_service.go
+++ b/backend/internal/services/container_registry_service.go
@@ -11,6 +11,7 @@ import (
"sync"
"time"
+ backoff "github.com/cenkalti/backoff/v5"
"golang.org/x/sync/singleflight"
cerrdefs "github.com/containerd/errdefs"
@@ -733,28 +734,73 @@ func (s *ContainerRegistryService) inspectImageDigestInternal(ctx context.Contex
return nil, err
}
- result, err := s.inspectImageDigestViaDaemonInternal(ctx, parts.NormalizedRef, parts.RegistryHost, externalCreds)
- if err == nil {
- return result, nil
- }
- if !isDistributionFallbackEligibleInternal(err) {
- return result, err
- }
+ var lastResult *registryDigestResult
+ var lastErr error
+ fallbackWarningLogged := false
+
+ bo := backoff.NewExponentialBackOff()
+ bo.InitialInterval = 500 * time.Millisecond
+ bo.MaxInterval = 10 * time.Second
+ bo.RandomizationFactor = 0.3
+
+ _, retryErr := backoff.Retry(ctx, func() (*registryDigestResult, error) {
+ result, err := s.inspectImageDigestViaDaemonInternal(ctx, parts.NormalizedRef, parts.RegistryHost, externalCreds)
+ if err == nil {
+ lastResult = result
+ return result, nil
+ }
+
+ if isRateLimitErrorInternal(err) {
+ lastResult = result
+ lastErr = err
+ slog.DebugContext(ctx, "rate limited by registry, will retry",
+ "registry", parts.RegistryHost,
+ "imageRef", parts.NormalizedRef)
+ return nil, err
+ }
+
+ if !isDistributionFallbackEligibleInternal(err) {
+ lastResult = result
+ lastErr = err
+ return nil, backoff.Permanent(err)
+ }
+
+ if !fallbackWarningLogged {
+ fallbackWarningLogged = true
+ slog.WarnContext(ctx, "distribution inspect unavailable, falling back to direct registry digest lookup",
+ "imageRef", parts.NormalizedRef,
+ "registry", parts.RegistryHost,
+ "error", err.Error())
+ }
+
+ fallbackResult, fallbackErr := s.inspectImageDigestViaRegistryInternal(ctx, parts.RegistryHost, parts.Repository, parts.Tag, externalCreds)
+ if fallbackErr == nil {
+ lastResult = fallbackResult
+ return fallbackResult, nil
+ }
+
+ if isRateLimitErrorInternal(fallbackErr) {
+ lastResult = fallbackResult
+ lastErr = fallbackErr
+ slog.DebugContext(ctx, "rate limited by registry on fallback, will retry",
+ "registry", parts.RegistryHost,
+ "imageRef", parts.NormalizedRef)
+ return nil, fallbackErr
+ }
- slog.WarnContext(ctx, "distribution inspect unavailable, falling back to direct registry digest lookup",
- "imageRef", parts.NormalizedRef,
- "registry", parts.RegistryHost,
- "error", err.Error())
+ lastResult = fallbackResult
+ lastErr = fmt.Errorf("daemon digest lookup failed; registry fallback failed: %w", errors.Join(err, fallbackErr))
+ return nil, backoff.Permanent(lastErr)
+ }, backoff.WithBackOff(bo), backoff.WithMaxTries(5))
- fallbackResult, fallbackErr := s.inspectImageDigestViaRegistryInternal(ctx, parts.RegistryHost, parts.Repository, parts.Tag, externalCreds)
- if fallbackErr == nil {
- return fallbackResult, nil
+ if retryErr != nil {
+ if errors.Is(retryErr, context.Canceled) || errors.Is(retryErr, context.DeadlineExceeded) {
+ return lastResult, retryErr
+ }
+ return lastResult, lastErr
}
- return fallbackResult, fmt.Errorf(
- "daemon digest lookup failed; registry fallback failed: %w",
- errors.Join(err, fallbackErr),
- )
+ return lastResult, nil
}
func (s *ContainerRegistryService) inspectImageDigestViaDaemonInternal(ctx context.Context, normalizedRef, registryHost string, externalCreds []containerregistry.Credential) (*registryDigestResult, error) {
@@ -1220,8 +1266,6 @@ func isUnauthorizedRegistryErrorInternal(err error) bool {
"no basic auth credentials",
"access denied",
"incorrect username or password",
- "toomanyrequests",
- "unauthenticated pull rate limit",
"status: 401",
"status 401",
"status: 403",
@@ -1237,6 +1281,27 @@ func isUnauthorizedRegistryErrorInternal(err error) bool {
return false
}
+func isRateLimitErrorInternal(err error) bool {
+ if err == nil {
+ return false
+ }
+ errLower := strings.ToLower(err.Error())
+ indicators := []string{
+ "toomanyrequests",
+ "rate limit",
+ "too many requests",
+ "status: 429",
+ "status 429",
+ "retry-after",
+ }
+ for _, indicator := range indicators {
+ if strings.Contains(errLower, indicator) {
+ return true
+ }
+ }
+ return false
+}
+
func isDockerHubRegistryInternal(registryHost string) bool {
return utilsregistry.NormalizeRegistryForComparison(registryHost) == "docker.io"
}
diff --git a/backend/internal/services/image_update_service.go b/backend/internal/services/image_update_service.go
index 5400af235..74a199683 100644
--- a/backend/internal/services/image_update_service.go
+++ b/backend/internal/services/image_update_service.go
@@ -13,6 +13,7 @@ import (
"github.com/getarcaneapp/arcane/backend/internal/models"
"github.com/getarcaneapp/arcane/backend/pkg/libarcane/crypto"
imageupdatecore "github.com/getarcaneapp/arcane/backend/pkg/libarcane/imageupdate"
+ "github.com/getarcaneapp/arcane/backend/pkg/libarcane/ratelimit"
registry "github.com/getarcaneapp/arcane/backend/pkg/libarcane/registryauth"
"github.com/getarcaneapp/arcane/types/containerregistry"
"github.com/getarcaneapp/arcane/types/imageupdate"
@@ -30,6 +31,7 @@ type ImageUpdateService struct {
dockerService *DockerClientService
eventService *EventService
notificationService *NotificationService
+ registryLimiter *ratelimit.RegistryRateLimiter
}
type ImageParts struct {
@@ -54,6 +56,7 @@ func NewImageUpdateService(db *database.DB, settingsService *SettingsService, re
dockerService: dockerService,
eventService: eventService,
notificationService: notificationService,
+ registryLimiter: ratelimit.NewRegistryRateLimiter(),
}
}
@@ -904,6 +907,17 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs
for _, img := range images {
g.Go(func() error {
+ registry := img.parts.Registry
+
+ if err := s.registryLimiter.Acquire(groupCtx, registry); err != nil {
+ slog.DebugContext(groupCtx, "skipping image check: registry limiter acquire failed",
+ "imageRef", img.canonicalRef,
+ "registry", registry,
+ "error", err.Error())
+ return err
+ }
+ defer s.registryLimiter.Release(registry)
+
res, snapshot := s.checkSingleImageInBatchInternal(groupCtx, resolvedCreds, img.parts)
mu.Lock()
@@ -921,6 +935,7 @@ func (s *ImageUpdateService) CheckMultipleImages(ctx context.Context, imageRefs
if err := g.Wait(); err != nil {
slog.ErrorContext(ctx, "Batch check error", "error", err)
+ return results, err
}
successCount, errorCount := countBatchResultOutcomesInternal(imageRefs, results)
diff --git a/backend/pkg/libarcane/ratelimit/registry_rate_limiter.go b/backend/pkg/libarcane/ratelimit/registry_rate_limiter.go
new file mode 100644
index 000000000..1a01bd9e1
--- /dev/null
+++ b/backend/pkg/libarcane/ratelimit/registry_rate_limiter.go
@@ -0,0 +1,89 @@
+package ratelimit
+
+import (
+ "context"
+ "strings"
+ "sync"
+
+ "golang.org/x/sync/semaphore"
+)
+
+type RegistryRateLimiter struct {
+ mu sync.RWMutex
+ semaphores map[string]*semaphore.Weighted
+ limits map[string]int64
+ defaultLimit int64
+}
+
+var defaultRegistryLimits = map[string]int64{
+ "ghcr.io": 3,
+ "docker.io": 5,
+ "quay.io": 4,
+ "gcr.io": 4,
+}
+
+const defaultRegistryConcurrencyLimit int64 = 3
+
+func NewRegistryRateLimiter() *RegistryRateLimiter {
+ return &RegistryRateLimiter{
+ semaphores: make(map[string]*semaphore.Weighted),
+ limits: defaultRegistryLimits,
+ defaultLimit: defaultRegistryConcurrencyLimit,
+ }
+}
+
+func (r *RegistryRateLimiter) Acquire(ctx context.Context, registry string) error {
+ sem := r.getSemaphoreInternal(registry)
+ return sem.Acquire(ctx, 1)
+}
+
+func (r *RegistryRateLimiter) Release(registry string) {
+ normalized := normalizeRegistryKeyInternal(registry)
+ r.mu.RLock()
+ sem, ok := r.semaphores[normalized]
+ r.mu.RUnlock()
+ if ok {
+ sem.Release(1)
+ }
+}
+
+func (r *RegistryRateLimiter) getSemaphoreInternal(registry string) *semaphore.Weighted {
+ normalized := normalizeRegistryKeyInternal(registry)
+
+ r.mu.RLock()
+ sem, ok := r.semaphores[normalized]
+ r.mu.RUnlock()
+ if ok {
+ return sem
+ }
+
+ r.mu.Lock()
+ defer r.mu.Unlock()
+
+ if sem, ok = r.semaphores[normalized]; ok {
+ return sem
+ }
+
+ limit := r.limitForRegistryInternal(normalized)
+ sem = semaphore.NewWeighted(limit)
+ r.semaphores[normalized] = sem
+ return sem
+}
+
+func (r *RegistryRateLimiter) limitForRegistryInternal(registry string) int64 {
+ if limit, ok := r.limits[registry]; ok {
+ return limit
+ }
+ if strings.Contains(registry, ".ecr.") && strings.Contains(registry, ".amazonaws.com") {
+ return 4
+ }
+ return r.defaultLimit
+}
+
+func normalizeRegistryKeyInternal(registry string) string {
+ normalized := strings.ToLower(strings.TrimSpace(registry))
+ if normalized == "registry-1.docker.io" || normalized == "index.docker.io" {
+ return "docker.io"
+ }
+ return normalized
+}