Skip to content

Modern Docker Management, Designed for Everyone

fix: image pulls failing for certain public images (#1855)

Kyle Mendellccad93682f72
diff --git a/backend/internal/services/image_service.go b/backend/internal/services/image_service.go
index e07031f61..68aa0381c 100644
--- a/backend/internal/services/image_service.go
+++ b/backend/internal/services/image_service.go
@@ -172,9 +172,18 @@ func (s *ImageService) PullImage(ctx context.Context, imageName string, progress
pullOptions = image.PullOptions{}
}
+ initialHasAuth := pullOptions.RegistryAuth != ""
+ retriedWithoutAuth := false
+
reader, err := dockerClient.ImagePull(ctx, imageName, pullOptions)
+ if err != nil && shouldRetryAnonymousPullInternal(pullOptions, err) {
+ retriedWithoutAuth = true
+ slog.WarnContext(ctx, "Docker ImagePull failed with registry auth; retrying anonymously", "image", imageName, "error", err.Error())
+ pullOptions = image.PullOptions{}
+ reader, err = dockerClient.ImagePull(ctx, imageName, pullOptions)
+ }
if err != nil {
- slog.ErrorContext(ctx, "Docker ImagePull failed", "image", imageName, "hasAuth", pullOptions.RegistryAuth != "", "error", err.Error())
+ slog.ErrorContext(ctx, "Docker ImagePull failed", "image", imageName, "hasAuth", pullOptions.RegistryAuth != "", "initialHasAuth", initialHasAuth, "retriedWithoutAuth", retriedWithoutAuth, "error", err.Error())
s.eventService.LogErrorEvent(ctx, models.EventTypeImageError, "image", "", imageName, user.ID, user.Username, "0", err, models.JSON{"action": "pull"})
return fmt.Errorf("failed to initiate image pull for %s: %w", imageName, err)
}
@@ -317,18 +326,29 @@ func (s *ImageService) getPullOptionsWithAuth(ctx context.Context, imageRef stri
for _, reg := range registries {
if s.isRegistryMatch(reg.URL, registryHost) {
+ username := strings.TrimSpace(reg.Username)
+ if username == "" {
+ slog.DebugContext(ctx, "Skipping database credentials for image pull", "registry", registryHost, "configuredRegistry", reg.URL, "reason", "missing_username")
+ continue
+ }
+
decryptedToken, err := s.registryService.GetDecryptedToken(ctx, reg.ID)
if err != nil {
return pullOptions, fmt.Errorf("failed to decrypt token for registry %s: %w", reg.URL, err)
}
+ token := strings.TrimSpace(decryptedToken)
+ if token == "" {
+ slog.DebugContext(ctx, "Skipping database credentials for image pull", "registry", registryHost, "configuredRegistry", reg.URL, "reason", "missing_token")
+ continue
+ }
- authStr, err := s.createAuthHeader(reg.Username, decryptedToken, s.normalizeRegistryURL(reg.URL))
+ authStr, err := s.createAuthHeader(username, token, s.normalizeRegistryURL(reg.URL))
if err != nil {
return pullOptions, fmt.Errorf("failed to create auth header: %w", err)
}
pullOptions.RegistryAuth = authStr
- slog.DebugContext(ctx, "Using database credentials for image pull", "registry", registryHost, "username", reg.Username)
+ slog.DebugContext(ctx, "Using database credentials for image pull", "registry", registryHost, "username", username)
break
}
}
@@ -402,6 +422,35 @@ func (s *ImageService) normalizeRegistryURL(url string) string {
return result
}
+func shouldRetryAnonymousPullInternal(pullOptions image.PullOptions, pullErr error) bool {
+ if pullOptions.RegistryAuth == "" || pullErr == nil {
+ return false
+ }
+ return isUnauthorizedPullErrorInternal(pullErr)
+}
+
+func isUnauthorizedPullErrorInternal(err error) bool {
+ if err == nil {
+ return false
+ }
+
+ errLower := strings.ToLower(err.Error())
+ unauthorizedIndicators := []string{
+ "unauthorized",
+ "authentication required",
+ "incorrect username or password",
+ "no basic auth credentials",
+ "access denied",
+ }
+
+ for _, indicator := range unauthorizedIndicators {
+ if strings.Contains(errLower, indicator) {
+ return true
+ }
+ }
+ return false
+}
+
func (s *ImageService) PruneImages(ctx context.Context, dangling bool) (*image.PruneReport, error) {
dockerClient, err := s.dockerService.GetClient()
if err != nil {
diff --git a/backend/internal/services/image_service_test.go b/backend/internal/services/image_service_test.go
index d3e913961..964e6f1f1 100644
--- a/backend/internal/services/image_service_test.go
+++ b/backend/internal/services/image_service_test.go
@@ -1,8 +1,22 @@
package services
import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
"testing"
+ glsqlite "github.com/glebarez/sqlite"
+ "github.com/stretchr/testify/require"
+ "gorm.io/gorm"
+
+ "github.com/docker/docker/api/types/image"
+ dockerregistry "github.com/docker/docker/api/types/registry"
+ "github.com/getarcaneapp/arcane/backend/internal/config"
+ "github.com/getarcaneapp/arcane/backend/internal/database"
+ "github.com/getarcaneapp/arcane/backend/internal/models"
+ "github.com/getarcaneapp/arcane/backend/internal/utils/crypto"
imagetypes "github.com/getarcaneapp/arcane/types/image"
"github.com/getarcaneapp/arcane/types/vulnerability"
"github.com/stretchr/testify/assert"
@@ -39,3 +53,95 @@ func TestApplyVulnerabilitySummariesToItemsInternal(t *testing.T) {
assert.Equal(t, summary, items[0].VulnerabilityScan)
assert.Nil(t, items[1].VulnerabilityScan)
}
+
+func setupImageServiceAuthTest(t *testing.T) (*ImageService, *database.DB) {
+ t.Helper()
+
+ db, err := gorm.Open(glsqlite.Open(":memory:"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, db.AutoMigrate(&models.ContainerRegistry{}))
+
+ crypto.InitEncryption(&config.Config{
+ Environment: config.AppEnvironmentTest,
+ EncryptionKey: "test-encryption-key-for-testing-32bytes-min",
+ })
+
+ dbWrap := &database.DB{DB: db}
+ svc := &ImageService{
+ registryService: NewContainerRegistryService(dbWrap),
+ }
+
+ return svc, dbWrap
+}
+
+func createTestPullRegistry(t *testing.T, db *database.DB, url, username, token string) {
+ t.Helper()
+
+ encryptedToken, err := crypto.Encrypt(token)
+ require.NoError(t, err)
+
+ reg := &models.ContainerRegistry{
+ URL: url,
+ Username: username,
+ Token: encryptedToken,
+ Enabled: true,
+ }
+ require.NoError(t, db.WithContext(context.Background()).Create(reg).Error)
+}
+
+func decodeRegistryAuth(t *testing.T, encoded string) dockerregistry.AuthConfig {
+ t.Helper()
+
+ raw, err := base64.StdEncoding.DecodeString(encoded)
+ require.NoError(t, err)
+
+ var cfg dockerregistry.AuthConfig
+ require.NoError(t, json.Unmarshal(raw, &cfg))
+ return cfg
+}
+
+func TestGetPullOptionsWithAuth_DBRegistrySkipsEmptyToken(t *testing.T) {
+ svc, db := setupImageServiceAuthTest(t)
+ createTestPullRegistry(t, db, "https://docker.io", "docker-user", " ")
+
+ pullOptions, err := svc.getPullOptionsWithAuth(context.Background(), "docker.io/library/nginx:latest", nil)
+ require.NoError(t, err)
+ assert.Empty(t, pullOptions.RegistryAuth)
+}
+
+func TestGetPullOptionsWithAuth_DBRegistrySkipsEmptyUsername(t *testing.T) {
+ svc, db := setupImageServiceAuthTest(t)
+ createTestPullRegistry(t, db, "https://docker.io", " ", "docker-token")
+
+ pullOptions, err := svc.getPullOptionsWithAuth(context.Background(), "docker.io/library/nginx:latest", nil)
+ require.NoError(t, err)
+ assert.Empty(t, pullOptions.RegistryAuth)
+}
+
+func TestGetPullOptionsWithAuth_DBRegistryUsesValidCredentials(t *testing.T) {
+ svc, db := setupImageServiceAuthTest(t)
+ createTestPullRegistry(t, db, "https://index.docker.io/v1/", "docker-user", "docker-token")
+
+ pullOptions, err := svc.getPullOptionsWithAuth(context.Background(), "docker.io/library/nginx:latest", nil)
+ require.NoError(t, err)
+ require.NotEmpty(t, pullOptions.RegistryAuth)
+
+ authCfg := decodeRegistryAuth(t, pullOptions.RegistryAuth)
+ assert.Equal(t, "docker-user", authCfg.Username)
+ assert.Equal(t, "docker-token", authCfg.Password)
+ assert.Equal(t, "https://index.docker.io/v1/", authCfg.ServerAddress)
+}
+
+func TestShouldRetryAnonymousPullInternal_UnauthorizedWithAuth(t *testing.T) {
+ err := errors.New(`Error response from daemon: Head "registry-1.docker.io/v2/library/nginx/manifests/latest": unauthorized: incorrect username or password`)
+
+ assert.True(t, shouldRetryAnonymousPullInternal(image.PullOptions{RegistryAuth: "encoded-auth"}, err))
+}
+
+func TestShouldRetryAnonymousPullInternal_SkipsRetryWithoutUnauthorizedOrAuth(t *testing.T) {
+ nonAuthErr := errors.New("Error response from daemon: i/o timeout")
+ unauthorizedErr := errors.New("unauthorized: authentication required")
+
+ assert.False(t, shouldRetryAnonymousPullInternal(image.PullOptions{RegistryAuth: "encoded-auth"}, nonAuthErr))
+ assert.False(t, shouldRetryAnonymousPullInternal(image.PullOptions{}, unauthorizedErr))
+}
diff --git a/backend/internal/services/updater_service.go b/backend/internal/services/updater_service.go
index afefc089d..fea7690ab 100644
--- a/backend/internal/services/updater_service.go
+++ b/backend/internal/services/updater_service.go
@@ -386,6 +386,8 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, dryRun bool) (*update
// UpdateSingleContainer updates a single container by ID to the latest available image.
// It pulls the new image, stops the container, removes it, and recreates it with the new image.
+//
+//nolint:gocognit // single-container update flow is intentionally linear with explicit early exits for failure reporting
func (s *UpdaterService) UpdateSingleContainer(ctx context.Context, containerID string) (*updater.Result, error) {
start := time.Now()
out := &updater.Result{Items: []updater.ResourceResult{}}
@@ -462,8 +464,32 @@ func (s *UpdaterService) UpdateSingleContainer(ctx context.Context, containerID
return out, nil
}
- // Get the image reference
- imageRef := targetContainer.Image
+ // Resolve the best pullable image reference for this container.
+ configImageRef := ""
+ if inspectBefore.Config != nil {
+ configImageRef = strings.TrimSpace(inspectBefore.Config.Image)
+ }
+ imageRef, imageRefSource := resolvePullableImageRefInternal(targetContainer.Image, configImageRef, nil)
+ if imageRef == "" && inspectBefore.Image != "" {
+ if imageInspect, inspectErr := dcli.ImageInspect(ctx, inspectBefore.Image); inspectErr == nil {
+ imageRef, imageRefSource = resolvePullableImageRefInternal(targetContainer.Image, configImageRef, imageInspect.RepoTags)
+ } else {
+ slog.DebugContext(ctx, "UpdateSingleContainer: failed to inspect container image for fallback refs", "containerID", containerID, "imageID", inspectBefore.Image, "error", inspectErr)
+ }
+ }
+ if imageRef == "" || isImageIDLikeReferenceInternal(imageRef) {
+ out.Items = append(out.Items, updater.ResourceResult{
+ ResourceID: targetContainer.ID,
+ ResourceType: "container",
+ ResourceName: containerName,
+ Status: "skipped",
+ Error: "unable to resolve a pullable image reference for container",
+ })
+ out.Skipped++
+ out.Duration = time.Since(start).String()
+ return out, nil
+ }
+
normalizedRef := s.normalizeRef(imageRef)
repo, tag := s.parseRepoAndTag(normalizedRef)
@@ -480,7 +506,7 @@ func (s *UpdaterService) UpdateSingleContainer(ctx context.Context, containerID
return out, nil
}
- slog.InfoContext(ctx, "UpdateSingleContainer: pulling new image", "containerID", containerID, "image", normalizedRef)
+ slog.InfoContext(ctx, "UpdateSingleContainer: pulling new image", "containerID", containerID, "image", normalizedRef, "imageRefSource", imageRefSource)
// Pull the latest image using the image service
if err := s.imageService.PullImage(ctx, normalizedRef, io.Discard, systemUser, nil); err != nil {
@@ -1399,6 +1425,26 @@ func (s *UpdaterService) resolveContainerImageMatchInternal(c container.Summary,
return "", ""
}
+func resolvePullableImageRefInternal(summaryImage, inspectConfigImage string, repoTags []string) (ref, source string) {
+ if image := strings.TrimSpace(inspectConfigImage); image != "" && !isImageIDLikeReferenceInternal(image) {
+ return image, "container_inspect_config"
+ }
+
+ if image := strings.TrimSpace(summaryImage); image != "" && !isImageIDLikeReferenceInternal(image) {
+ return image, "container_summary"
+ }
+
+ for _, tag := range repoTags {
+ trimmed := strings.TrimSpace(tag)
+ if trimmed == "" || trimmed == "<none>:<none>" || isImageIDLikeReferenceInternal(trimmed) {
+ continue
+ }
+ return trimmed, "image_repo_tag"
+ }
+
+ return "", ""
+}
+
// parseNormalizedRef expects a normalized ref in the form "host/repository:tag".
func (s *UpdaterService) parseNormalizedRef(ref string) (host, repository, tag string) {
// host/repo:tag
diff --git a/backend/internal/services/updater_service_test.go b/backend/internal/services/updater_service_test.go
index 0cce3fb63..9a2d11190 100644
--- a/backend/internal/services/updater_service_test.go
+++ b/backend/internal/services/updater_service_test.go
@@ -474,6 +474,56 @@ func TestResolveContainerImageMatchInternal(t *testing.T) {
}
}
+func TestResolvePullableImageRefInternal(t *testing.T) {
+ tests := []struct {
+ name string
+ summaryImage string
+ inspectImage string
+ repoTags []string
+ wantRef string
+ wantSource string
+ }{
+ {
+ name: "prefers inspect config image",
+ summaryImage: "portainer/portainer.ce:latest",
+ inspectImage: "ghcr.io/example/app:1.2.3",
+ wantRef: "ghcr.io/example/app:1.2.3",
+ wantSource: "container_inspect_config",
+ },
+ {
+ name: "falls back to summary image when inspect image is id-like",
+ summaryImage: "portainer/portainer.ce:latest",
+ inspectImage: "sha256:abcdef",
+ wantRef: "portainer/portainer.ce:latest",
+ wantSource: "container_summary",
+ },
+ {
+ name: "falls back to repo tag when summary and inspect are id-like",
+ summaryImage: "sha256:abc123",
+ inspectImage: "sha256:def456",
+ repoTags: []string{"<none>:<none>", "docker.io/library/portainer/portainer.ce:latest"},
+ wantRef: "docker.io/library/portainer/portainer.ce:latest",
+ wantSource: "image_repo_tag",
+ },
+ {
+ name: "returns empty when only id-like candidates exist",
+ summaryImage: "sha256:abc123",
+ inspectImage: "sha256:def456",
+ repoTags: []string{"<none>:<none>", "sha256:fff"},
+ wantRef: "",
+ wantSource: "",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ gotRef, gotSource := resolvePullableImageRefInternal(tt.summaryImage, tt.inspectImage, tt.repoTags)
+ assert.Equal(t, tt.wantRef, gotRef)
+ assert.Equal(t, tt.wantSource, gotSource)
+ })
+ }
+}
+
func TestUpdaterService_StatusTrackingInternal(t *testing.T) {
svc := &UpdaterService{
updatingContainers: map[string]bool{},
diff --git a/backend/pkg/scheduler/auto_heal_job.go b/backend/pkg/scheduler/auto_heal_job.go
index c03c98af5..db958f610 100644
--- a/backend/pkg/scheduler/auto_heal_job.go
+++ b/backend/pkg/scheduler/auto_heal_job.go
@@ -213,7 +213,7 @@ func (j *AutoHealJob) parseExcludedContainers(ctx context.Context) map[string]st
if raw == "" {
return excluded
}
- for _, name := range strings.Split(raw, ",") {
+ for name := range strings.SplitSeq(raw, ",") {
trimmed := strings.TrimSpace(name)
if trimmed != "" {
excluded[trimmed] = struct{}{}
diff --git a/backend/pkg/scheduler/auto_heal_job_test.go b/backend/pkg/scheduler/auto_heal_job_test.go
index 9b3c05176..ec3222862 100644
--- a/backend/pkg/scheduler/auto_heal_job_test.go
+++ b/backend/pkg/scheduler/auto_heal_job_test.go
@@ -20,7 +20,7 @@ func TestAutoHeal_CanRestart_UnderLimit(t *testing.T) {
require.True(t, job.CanRestartExported("container-1", 5, 30*time.Minute))
// Record 4 restarts (under limit of 5)
- for i := 0; i < 4; i++ {
+ for range 4 {
job.RecordRestartExported("container-1")
}
@@ -31,7 +31,7 @@ func TestAutoHeal_CanRestart_AtLimit(t *testing.T) {
job := newTestAutoHealJob()
// Record exactly 5 restarts (at limit)
- for i := 0; i < 5; i++ {
+ for range 5 {
job.RecordRestartExported("container-1")
}
@@ -43,7 +43,7 @@ func TestAutoHeal_CanRestart_WindowExpiry(t *testing.T) {
// Record 5 restarts 31 minutes ago (outside window)
oldTime := time.Now().Add(-31 * time.Minute)
- for i := 0; i < 5; i++ {
+ for range 5 {
job.RecordRestartAtExported("container-1", oldTime)
}
@@ -56,12 +56,12 @@ func TestAutoHeal_CanRestart_MixedTimestamps(t *testing.T) {
// Record 3 old restarts (outside window)
oldTime := time.Now().Add(-31 * time.Minute)
- for i := 0; i < 3; i++ {
+ for range 3 {
job.RecordRestartAtExported("container-1", oldTime)
}
// Record 4 recent restarts (inside window)
- for i := 0; i < 4; i++ {
+ for range 4 {
job.RecordRestartExported("container-1")
}
@@ -79,7 +79,7 @@ func TestAutoHeal_CanRestart_DifferentContainers(t *testing.T) {
job := newTestAutoHealJob()
// Fill up container-1
- for i := 0; i < 5; i++ {
+ for range 5 {
job.RecordRestartExported("container-1")
}
@@ -101,7 +101,7 @@ func TestAutoHeal_ResetRestartTracking(t *testing.T) {
job := newTestAutoHealJob()
// Fill up container-1
- for i := 0; i < 5; i++ {
+ for range 5 {
job.RecordRestartExported("container-1")
}
require.False(t, job.CanRestartExported("container-1", 5, 30*time.Minute))