Skip to content

Modern Docker Management, Designed for Everyone

fix: image update inspection fallback to manual vs using mobys distribution inspect (#2080)

Kyle Mendell73da7cd1833f
diff --git a/backend/internal/services/container_registry_service.go b/backend/internal/services/container_registry_service.go
index 78ec013c4..732f1e0eb 100644
--- a/backend/internal/services/container_registry_service.go
+++ b/backend/internal/services/container_registry_service.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"log/slog"
+ "net/http"
"strings"
"sync"
"time"
@@ -18,10 +19,10 @@ import (
"github.com/getarcaneapp/arcane/backend/internal/utils/mapper"
"github.com/getarcaneapp/arcane/backend/internal/utils/pagination"
utilsregistry "github.com/getarcaneapp/arcane/backend/internal/utils/registry"
+ utilsdistribution "github.com/getarcaneapp/arcane/backend/pkg/utils/distribution"
"github.com/getarcaneapp/arcane/types/containerregistry"
dockerregistry "github.com/moby/moby/api/types/registry"
"github.com/moby/moby/client"
- ref "go.podman.io/image/v5/docker/reference"
)
const (
@@ -50,17 +51,19 @@ type resolvedRegistryCredential struct {
}
type ContainerRegistryService struct {
- db *database.DB
- dockerClient registryDaemonGetter
- cache map[string]*cache.Cache[string] // imageRef -> digest cache
- cacheMu sync.RWMutex
+ db *database.DB
+ dockerClient registryDaemonGetter
+ distributionHTTPClient *http.Client
+ cache map[string]*cache.Cache[string] // imageRef -> digest cache
+ cacheMu sync.RWMutex
}
func NewContainerRegistryService(db *database.DB, dockerClient registryDaemonGetter) *ContainerRegistryService {
return &ContainerRegistryService{
- db: db,
- dockerClient: dockerClient,
- cache: make(map[string]*cache.Cache[string]),
+ db: db,
+ dockerClient: dockerClient,
+ distributionHTTPClient: utilsdistribution.NewRegistryHTTPClient(),
+ cache: make(map[string]*cache.Cache[string]),
}
}
@@ -346,11 +349,36 @@ func (s *ContainerRegistryService) GetImageDigest(ctx context.Context, imageRef
}
func (s *ContainerRegistryService) inspectImageDigestInternal(ctx context.Context, imageRef string, externalCreds []containerregistry.Credential) (*registryDigestResult, error) {
- normalizedRef, registryHost, err := normalizeImageReferenceForDistributionInternal(imageRef)
+ parts, err := normalizeDistributionReferenceInternal(imageRef)
if err != nil {
return nil, err
}
+ result, err := s.inspectImageDigestViaDaemonInternal(ctx, parts.NormalizedRef, parts.RegistryHost, externalCreds)
+ if err == nil {
+ return result, nil
+ }
+ if !isDistributionFallbackEligibleInternal(err) {
+ return result, err
+ }
+
+ slog.WarnContext(ctx, "distribution inspect unavailable, falling back to direct registry digest lookup",
+ "imageRef", parts.NormalizedRef,
+ "registry", parts.RegistryHost,
+ "error", err.Error())
+
+ fallbackResult, fallbackErr := s.inspectImageDigestViaRegistryInternal(ctx, parts.RegistryHost, parts.Repository, parts.Tag, externalCreds)
+ if fallbackErr == nil {
+ return fallbackResult, nil
+ }
+
+ return fallbackResult, fmt.Errorf(
+ "daemon digest lookup failed; registry fallback failed: %w",
+ errors.Join(err, fallbackErr),
+ )
+}
+
+func (s *ContainerRegistryService) inspectImageDigestViaDaemonInternal(ctx context.Context, normalizedRef, registryHost string, externalCreds []containerregistry.Credential) (*registryDigestResult, error) {
dockerClient, err := s.getDockerClientInternal(ctx)
if err != nil {
return nil, err
@@ -375,7 +403,8 @@ func (s *ContainerRegistryService) inspectImageDigestInternal(ctx context.Contex
credentials, credErr := s.getMatchingRegistryCredentialsInternal(ctx, registryHost, externalCreds)
if credErr != nil {
- return &registryDigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}, credErr
+ return &registryDigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost},
+ fmt.Errorf("distribution inspect: anonymous access unauthorized; credential lookup failed: %w", errors.Join(err, credErr))
}
lastErr := err
@@ -423,6 +452,55 @@ func (s *ContainerRegistryService) inspectImageDigestInternal(ctx context.Contex
return partial, fmt.Errorf("distribution inspect failed for %s: %w", normalizedRef, lastErr)
}
+func (s *ContainerRegistryService) inspectImageDigestViaRegistryInternal(ctx context.Context, registryHost, repository, tag string, externalCreds []containerregistry.Credential) (*registryDigestResult, error) {
+ digest, err := s.fetchDigestFromRegistryInternal(ctx, registryHost, repository, tag, nil)
+ if err == nil {
+ return &registryDigestResult{
+ Digest: digest,
+ AuthMethod: "anonymous",
+ AuthRegistry: registryHost,
+ }, nil
+ }
+ if !isUnauthorizedRegistryErrorInternal(err) {
+ return &registryDigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost},
+ fmt.Errorf("registry manifest inspect failed for %s/%s:%s: %w", registryHost, repository, tag, err)
+ }
+
+ credentials, credErr := s.getMatchingRegistryCredentialsInternal(ctx, registryHost, externalCreds)
+ if credErr != nil {
+ return &registryDigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost},
+ fmt.Errorf("registry manifest inspect: anonymous access unauthorized; credential lookup failed: %w", errors.Join(err, credErr))
+ }
+
+ lastErr := err
+ var lastCred resolvedRegistryCredential
+ for _, credential := range credentials {
+ lastCred = credential
+
+ digest, err = s.fetchDigestFromRegistryInternal(ctx, registryHost, repository, tag, &credential)
+ if err == nil {
+ return &registryDigestResult{
+ Digest: digest,
+ AuthMethod: "credential",
+ AuthUsername: credential.Username,
+ AuthRegistry: registryHost,
+ UsedCredential: true,
+ }, nil
+ }
+
+ lastErr = err
+ }
+
+ partial := &registryDigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}
+ if lastCred.Username != "" {
+ partial.AuthMethod = "credential"
+ partial.AuthUsername = lastCred.Username
+ partial.UsedCredential = true
+ }
+
+ return partial, fmt.Errorf("registry manifest inspect failed for %s/%s:%s: %w", registryHost, repository, tag, lastErr)
+}
+
func (s *ContainerRegistryService) getDockerClientInternal(ctx context.Context) (RegistryDaemonClient, error) {
if s.dockerClient == nil {
return nil, errors.New("docker client unavailable")
@@ -608,25 +686,17 @@ func (s *ContainerRegistryService) deleteUnsyncedInternal(ctx context.Context, e
return nil
}
+func normalizeDistributionReferenceInternal(imageRef string) (*utilsdistribution.Reference, error) {
+ return utilsdistribution.NormalizeReference(imageRef)
+}
+
func normalizeImageReferenceForDistributionInternal(imageRef string) (string, string, error) {
- named, err := ref.ParseNormalizedNamed(strings.TrimSpace(imageRef))
+ parts, err := normalizeDistributionReferenceInternal(imageRef)
if err != nil {
- return "", "", fmt.Errorf("invalid image reference %q: %w", imageRef, err)
- }
-
- if _, ok := named.(ref.Digested); ok {
- return "", "", fmt.Errorf("digest-pinned references are not supported for distribution inspect: %q", imageRef)
+ return "", "", err
}
- registryHost := utilsregistry.NormalizeRegistryForComparison(ref.Domain(named))
- repository := ref.Path(named)
-
- tag := "latest"
- if tagged, ok := named.(ref.NamedTagged); ok {
- tag = tagged.Tag()
- }
-
- return registryHost + "/" + repository + ":" + tag, registryHost, nil
+ return parts.NormalizedRef, parts.RegistryHost, nil
}
func normalizeRegistryServerAddressInternal(registryURL string) string {
@@ -662,6 +732,10 @@ func isUnauthorizedRegistryErrorInternal(err error) bool {
"no basic auth credentials",
"access denied",
"incorrect username or password",
+ "status: 401",
+ "status 401",
+ "status: 403",
+ "status 403",
}
for _, indicator := range indicators {
@@ -672,3 +746,30 @@ func isUnauthorizedRegistryErrorInternal(err error) bool {
return false
}
+
+func isDistributionFallbackEligibleInternal(err error) bool {
+ if err == nil {
+ return false
+ }
+
+ return utilsdistribution.IsFallbackEligibleDaemonError(err)
+}
+
+func (s *ContainerRegistryService) fetchDigestFromRegistryInternal(ctx context.Context, registryHost, repository, tag string, credential *resolvedRegistryCredential) (string, error) {
+ var distributionCredential *utilsdistribution.Credentials
+ if credential != nil {
+ distributionCredential = &utilsdistribution.Credentials{
+ Username: strings.TrimSpace(credential.Username),
+ Token: strings.TrimSpace(credential.Token),
+ }
+ }
+
+ return utilsdistribution.FetchDigestWithHTTPClient(
+ ctx,
+ registryHost,
+ repository,
+ tag,
+ distributionCredential,
+ s.distributionHTTPClient,
+ )
+}
diff --git a/backend/internal/services/container_registry_service_test.go b/backend/internal/services/container_registry_service_test.go
index a38c0e2bb..316374f3b 100644
--- a/backend/internal/services/container_registry_service_test.go
+++ b/backend/internal/services/container_registry_service_test.go
@@ -2,7 +2,12 @@ package services
import (
"context"
+ "encoding/json"
"errors"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
"testing"
dockerregistry "github.com/moby/moby/api/types/registry"
@@ -18,6 +23,12 @@ type fakeRegistryDaemonClient struct {
distributionInspectFn func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error)
}
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+ return f(req)
+}
+
func (f *fakeRegistryDaemonClient) RegistryLogin(ctx context.Context, options client.RegistryLoginOptions) (client.RegistryLoginResult, error) {
if f.registryLoginFn == nil {
return client.RegistryLoginResult{}, nil
@@ -32,6 +43,29 @@ func (f *fakeRegistryDaemonClient) DistributionInspect(ctx context.Context, imag
return f.distributionInspectFn(ctx, imageRef, options)
}
+func newTestDockerClient(t *testing.T, server *httptest.Server) *client.Client {
+ t.Helper()
+
+ httpClient := server.Client()
+ cli, err := client.New(
+ client.WithHost(server.URL),
+ client.WithVersion("1.41"),
+ client.WithHTTPClient(httpClient),
+ )
+ require.NoError(t, err)
+
+ t.Cleanup(func() {
+ _ = cli.Close()
+ })
+
+ return cli
+}
+
+func TestNewContainerRegistryService_InitializesDistributionHTTPClient(t *testing.T) {
+ svc := NewContainerRegistryService(nil, nil)
+ require.NotNil(t, svc.distributionHTTPClient)
+}
+
func TestContainerRegistryService_GetAllRegistryAuthConfigs_NormalizesHosts(t *testing.T) {
_, db := setupImageServiceAuthTest(t)
createTestPullRegistry(t, db, "https://index.docker.io/v1/", "docker-user", "docker-token")
@@ -190,3 +224,230 @@ func TestContainerRegistryService_InspectImageDigest_RetriesWithStoredCredential
assert.Equal(t, "docker-user", result.AuthUsername)
assert.True(t, result.UsedCredential)
}
+
+func TestContainerRegistryService_InspectImageDigest_FallsBackWhenDistributionNotFound(t *testing.T) {
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/v2/team/app/manifests/1.2.3" {
+ w.Header().Set("Docker-Content-Digest", "sha256:fallback404")
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+
+ serverURL, err := url.Parse(server.URL)
+ require.NoError(t, err)
+
+ var calls int
+ svc := NewContainerRegistryService(nil, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ calls++
+ assert.Equal(t, serverURL.Host+"/team/app:1.2.3", imageRef)
+ assert.Empty(t, options.EncodedRegistryAuth)
+ return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
+ },
+ }, nil
+ })
+ svc.distributionHTTPClient = server.Client()
+
+ result, err := svc.inspectImageDigestInternal(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
+ require.NoError(t, err)
+ assert.Equal(t, 1, calls)
+ assert.Equal(t, "sha256:fallback404", result.Digest)
+ assert.Equal(t, "anonymous", result.AuthMethod)
+ assert.Equal(t, serverURL.Host, result.AuthRegistry)
+}
+
+func TestContainerRegistryService_InspectImageDigest_FallsBackWhenDistributionForbidden(t *testing.T) {
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/v2/team/app/manifests/1.2.3" {
+ w.Header().Set("Docker-Content-Digest", "sha256:fallback403")
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ http.NotFound(w, r)
+ }))
+ defer server.Close()
+
+ serverURL, err := url.Parse(server.URL)
+ require.NoError(t, err)
+
+ var calls int
+ svc := NewContainerRegistryService(nil, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ calls++
+ assert.Equal(t, serverURL.Host+"/team/app:1.2.3", imageRef)
+ assert.Empty(t, options.EncodedRegistryAuth)
+ return client.DistributionInspectResult{}, errors.New("Error response from daemon: <html><body><h1>403 Forbidden</h1> Request forbidden by administrative rules. </body></html>")
+ },
+ }, nil
+ })
+ svc.distributionHTTPClient = server.Client()
+
+ result, err := svc.inspectImageDigestInternal(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
+ require.NoError(t, err)
+ assert.Equal(t, 1, calls)
+ assert.Equal(t, "sha256:fallback403", result.Digest)
+ assert.Equal(t, "anonymous", result.AuthMethod)
+ assert.Equal(t, serverURL.Host, result.AuthRegistry)
+}
+
+func TestContainerRegistryService_InspectImageDigest_RetriesStoredCredentialsAfterRegistryAuth403(t *testing.T) {
+ _, db := setupImageServiceAuthTest(t)
+
+ var authHeaders []string
+ var tokenURL string
+ handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/v2/team/app/manifests/1.2.3":
+ authHeaders = append(authHeaders, r.Header.Get("Authorization"))
+ switch len(authHeaders) {
+ case 1:
+ w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenURL+`",service="registry.example.com"`)
+ w.WriteHeader(http.StatusUnauthorized)
+ case 2:
+ w.WriteHeader(http.StatusForbidden)
+ case 3:
+ w.Header().Set("Docker-Content-Digest", "sha256:stored-credential")
+ w.WriteHeader(http.StatusOK)
+ default:
+ t.Fatalf("unexpected manifest call %d", len(authHeaders))
+ }
+ case "/token":
+ username, password, ok := r.BasicAuth()
+ if !ok {
+ require.Equal(t, "", r.Header.Get("Authorization"))
+ require.NoError(t, json.NewEncoder(w).Encode(map[string]string{
+ "token": "anonymous-token",
+ }))
+ return
+ }
+
+ require.Equal(t, "stored-user", username)
+ require.Equal(t, "stored-token", password)
+ require.NoError(t, json.NewEncoder(w).Encode(map[string]string{
+ "token": "credential-token",
+ }))
+ default:
+ http.NotFound(w, r)
+ }
+ })
+ server := httptest.NewTLSServer(handler)
+ defer server.Close()
+ tokenURL = server.URL + "/token"
+
+ serverURL, err := url.Parse(server.URL)
+ require.NoError(t, err)
+ createTestPullRegistry(t, db, server.URL, "stored-user", "stored-token")
+
+ svc := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
+ },
+ }, nil
+ })
+ svc.distributionHTTPClient = server.Client()
+
+ result, err := svc.inspectImageDigestInternal(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
+ require.NoError(t, err)
+ assert.Equal(t, "sha256:stored-credential", result.Digest)
+ assert.Equal(t, "credential", result.AuthMethod)
+ assert.Equal(t, "stored-user", result.AuthUsername)
+ assert.True(t, result.UsedCredential)
+ require.Len(t, authHeaders, 3)
+ assert.Equal(t, "", authHeaders[0])
+ assert.Equal(t, "Bearer anonymous-token", authHeaders[1])
+ assert.Equal(t, "Basic c3RvcmVkLXVzZXI6c3RvcmVkLXRva2Vu", authHeaders[2])
+}
+
+func TestContainerRegistryService_InspectImageDigest_DoesNotFallbackOnTLSFailure(t *testing.T) {
+ svc := NewContainerRegistryService(nil, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ assert.Equal(t, "registry.example.com/team/app:1.2.3", imageRef)
+ assert.Empty(t, options.EncodedRegistryAuth)
+ return client.DistributionInspectResult{}, errors.New("tls: failed to verify certificate: x509: certificate signed by unknown authority")
+ },
+ }, nil
+ })
+
+ result, err := svc.inspectImageDigestInternal(context.Background(), "registry.example.com/team/app:1.2.3", nil)
+ require.Error(t, err)
+ require.NotNil(t, result)
+ assert.Contains(t, strings.ToLower(err.Error()), "x509")
+ assert.NotContains(t, err.Error(), "registry fallback failed")
+ assert.Equal(t, "anonymous", result.AuthMethod)
+ assert.Equal(t, "registry.example.com", result.AuthRegistry)
+}
+
+func TestContainerRegistryService_InspectImageDigest_PreservesDaemonAndFallbackErrors(t *testing.T) {
+ daemonErr := errors.New("Error response from daemon: Not Found")
+ fallbackErr := errors.New("dial tcp: i/o timeout")
+
+ svc := NewContainerRegistryService(nil, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ return client.DistributionInspectResult{}, daemonErr
+ },
+ }, nil
+ })
+ svc.distributionHTTPClient = &http.Client{
+ Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ return nil, fallbackErr
+ }),
+ }
+
+ result, err := svc.inspectImageDigestInternal(context.Background(), "registry.example.com/team/app:1.2.3", nil)
+ require.Error(t, err)
+ require.NotNil(t, result)
+ assert.ErrorIs(t, err, daemonErr)
+ assert.ErrorIs(t, err, fallbackErr)
+}
+
+func TestContainerRegistryService_InspectImageDigest_PreservesAnonymousUnauthorizedWhenCredentialLookupFails(t *testing.T) {
+ _, db := setupImageServiceAuthTest(t)
+ sqlDB, err := db.DB.DB()
+ require.NoError(t, err)
+ require.NoError(t, sqlDB.Close())
+
+ var tokenURL string
+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/v2/team/app/manifests/1.2.3":
+ w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenURL+`",service="registry.example.com"`)
+ w.WriteHeader(http.StatusUnauthorized)
+ case "/token":
+ require.NoError(t, json.NewEncoder(w).Encode(map[string]string{
+ "token": "anonymous-token",
+ }))
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ tokenURL = server.URL + "/token"
+
+ serverURL, err := url.Parse(server.URL)
+ require.NoError(t, err)
+
+ svc := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
+ },
+ }, nil
+ })
+ svc.distributionHTTPClient = server.Client()
+
+ result, err := svc.inspectImageDigestInternal(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
+ require.Error(t, err)
+ require.NotNil(t, result)
+ assert.Equal(t, "anonymous", result.AuthMethod)
+ assert.Contains(t, err.Error(), "anonymous access unauthorized")
+ assert.Contains(t, err.Error(), "status: 401")
+ assert.Contains(t, err.Error(), "failed to load enabled registries")
+}
diff --git a/backend/internal/services/image_update_service_test.go b/backend/internal/services/image_update_service_test.go
index 6d6a9eb6d..08ca6afd5 100644
--- a/backend/internal/services/image_update_service_test.go
+++ b/backend/internal/services/image_update_service_test.go
@@ -2,7 +2,13 @@ package services
import (
"context"
+ "encoding/json"
+ "errors"
"fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
"testing"
"time"
@@ -11,6 +17,7 @@ import (
"github.com/getarcaneapp/arcane/types/imageupdate"
glsqlite "github.com/glebarez/sqlite"
dockertypesimage "github.com/moby/moby/api/types/image"
+ "github.com/moby/moby/client"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
ref "go.podman.io/image/v5/docker/reference"
@@ -349,10 +356,125 @@ func setupImageUpdateTestDB(t *testing.T) *database.DB {
dsn := fmt.Sprintf("file:image-update-test-%d?mode=memory&cache=shared", time.Now().UnixNano())
db, err := gorm.Open(glsqlite.Open(dsn), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&models.ImageUpdateRecord{}))
+ require.NoError(t, db.AutoMigrate(&models.ImageUpdateRecord{}, &models.Event{}))
return &database.DB{DB: db}
}
+func newImageUpdateFallbackServer(t *testing.T, repositoryTag, localDigest, remoteDigest string) *httptest.Server {
+ t.Helper()
+
+ repository := repositoryTag
+ tag := "latest"
+ if tagIndex := strings.LastIndex(repositoryTag, ":"); tagIndex > strings.LastIndex(repositoryTag, "/") {
+ repository = repositoryTag[:tagIndex]
+ tag = repositoryTag[tagIndex+1:]
+ }
+ manifestPath := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag)
+
+ return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case strings.Contains(r.URL.Path, "/images/") && strings.HasSuffix(r.URL.Path, "/json"):
+ imageRef := r.Host + "/" + repositoryTag
+ repositoryRef := imageRef
+ if tagIndex := strings.LastIndex(imageRef, ":"); tagIndex > strings.LastIndex(imageRef, "/") {
+ repositoryRef = imageRef[:tagIndex]
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ require.NoError(t, json.NewEncoder(w).Encode(dockertypesimage.InspectResponse{
+ ID: "sha256:local-image-id",
+ RepoTags: []string{imageRef},
+ RepoDigests: []string{repositoryRef + "@" + localDigest},
+ }))
+ return
+ case r.URL.Path == manifestPath:
+ w.Header().Set("Docker-Content-Digest", remoteDigest)
+ w.WriteHeader(http.StatusOK)
+ return
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+}
+
+func TestImageUpdateService_CheckImageUpdate_UsesRegistryFallback(t *testing.T) {
+ db := setupImageUpdateTestDB(t)
+
+ server := newImageUpdateFallbackServer(t, "team/app:1.2.3", "sha256:localdigest", "sha256:remotedigest")
+ defer server.Close()
+
+ serverURL, err := url.Parse(server.URL)
+ require.NoError(t, err)
+ imageRef := serverURL.Host + "/team/app:1.2.3"
+
+ registryService := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
+ },
+ }, nil
+ })
+ registryService.distributionHTTPClient = server.Client()
+
+ dockerService := &DockerClientService{client: newTestDockerClient(t, server)}
+ eventService := NewEventService(db, nil, nil)
+ svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil)
+
+ result, err := svc.CheckImageUpdate(context.Background(), imageRef)
+ require.NoError(t, err)
+ require.NotNil(t, result)
+ assert.True(t, result.HasUpdate)
+ assert.Equal(t, "digest", result.UpdateType)
+ assert.Equal(t, "sha256:localdigest", result.CurrentDigest)
+ assert.Equal(t, "sha256:remotedigest", result.LatestDigest)
+ assert.Equal(t, "anonymous", result.AuthMethod)
+ assert.Equal(t, serverURL.Host, result.AuthRegistry)
+
+ var saved models.ImageUpdateRecord
+ require.NoError(t, db.WithContext(context.Background()).Where("id = ?", "sha256:local-image-id").First(&saved).Error)
+ assert.Equal(t, "sha256:remotedigest", stringPtrToString(saved.LatestDigest))
+}
+
+func TestImageUpdateService_CheckMultipleImages_UsesRegistryFallback(t *testing.T) {
+ db := setupImageUpdateTestDB(t)
+
+ server := newImageUpdateFallbackServer(t, "team/app:1.2.3", "sha256:batchlocal", "sha256:batchremote")
+ defer server.Close()
+
+ serverURL, err := url.Parse(server.URL)
+ require.NoError(t, err)
+ imageRef := serverURL.Host + "/team/app:1.2.3"
+
+ registryService := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ return client.DistributionInspectResult{}, errors.New("Error response from daemon: <html><body><h1>403 Forbidden</h1> Request forbidden by administrative rules. </body></html>")
+ },
+ }, nil
+ })
+ registryService.distributionHTTPClient = server.Client()
+
+ dockerService := &DockerClientService{client: newTestDockerClient(t, server)}
+ eventService := NewEventService(db, nil, nil)
+ svc := NewImageUpdateService(db, nil, registryService, dockerService, eventService, nil)
+
+ results, err := svc.CheckMultipleImages(context.Background(), []string{imageRef}, nil)
+ require.NoError(t, err)
+ require.Contains(t, results, imageRef)
+
+ result := results[imageRef]
+ require.NotNil(t, result)
+ assert.True(t, result.HasUpdate)
+ assert.Equal(t, "sha256:batchlocal", result.CurrentDigest)
+ assert.Equal(t, "sha256:batchremote", result.LatestDigest)
+ assert.Equal(t, "anonymous", result.AuthMethod)
+ assert.Equal(t, serverURL.Host, result.AuthRegistry)
+
+ var saved models.ImageUpdateRecord
+ require.NoError(t, db.WithContext(context.Background()).Where("id = ?", "sha256:local-image-id").First(&saved).Error)
+ assert.Equal(t, "sha256:batchremote", stringPtrToString(saved.LatestDigest))
+}
+
// TestNotificationSentLogic tests the notification_sent flag behavior
func TestImageUpdateService_NotificationSentLogic(t *testing.T) {
db := setupImageUpdateTestDB(t)
diff --git a/backend/pkg/utils/distribution/distribution.go b/backend/pkg/utils/distribution/distribution.go
new file mode 100644
index 000000000..258fb61f6
--- /dev/null
+++ b/backend/pkg/utils/distribution/distribution.go
@@ -0,0 +1,421 @@
+package distribution
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ ref "go.podman.io/image/v5/docker/reference"
+)
+
+const defaultRegistryHost = "index.docker.io"
+
+type Credentials struct {
+ Username string
+ Token string
+}
+
+type Reference struct {
+ NormalizedRef string
+ RegistryHost string
+ Repository string
+ Tag string
+}
+
+func NormalizeReference(imageRef string) (*Reference, error) {
+ named, err := ref.ParseNormalizedNamed(strings.TrimSpace(imageRef))
+ if err != nil {
+ return nil, fmt.Errorf("invalid image reference %q: %w", imageRef, err)
+ }
+
+ if _, ok := named.(ref.Digested); ok {
+ return nil, fmt.Errorf("digest-pinned references are not supported for distribution inspect: %q", imageRef)
+ }
+
+ registryHost := normalizeRegistryForComparisonInternal(ref.Domain(named))
+ repository := ref.Path(named)
+
+ tag := "latest"
+ if tagged, ok := named.(ref.NamedTagged); ok {
+ tag = tagged.Tag()
+ }
+
+ return &Reference{
+ NormalizedRef: registryHost + "/" + repository + ":" + tag,
+ RegistryHost: registryHost,
+ Repository: repository,
+ Tag: tag,
+ }, nil
+}
+
+func IsFallbackEligibleDaemonError(err error) bool {
+ if err == nil {
+ return false
+ }
+
+ errLower := strings.ToLower(err.Error())
+ if strings.Contains(errLower, "unauthorized") ||
+ strings.Contains(errLower, "authentication required") ||
+ strings.Contains(errLower, "no basic auth credentials") ||
+ strings.Contains(errLower, "access denied") ||
+ strings.Contains(errLower, "incorrect username or password") ||
+ strings.Contains(errLower, "status: 401") ||
+ strings.Contains(errLower, "status 401") {
+ return false
+ }
+
+ if strings.Contains(errLower, "x509") || strings.Contains(errLower, "certificate") || strings.Contains(errLower, "tls") {
+ return false
+ }
+
+ // Network-level daemon errors are intentionally not fallback-eligible.
+ // If the daemon cannot reach the registry at all, the backend's direct HTTP
+ // client is also unlikely to succeed, so only registry/API capability
+ // failures are allowed to trigger fallback.
+ indicators := []string{
+ "not found",
+ " 404 ",
+ "status: 404",
+ "status 404",
+ "403 forbidden",
+ "status: 403",
+ "status 403",
+ "administrative rules",
+ "not implemented",
+ "unsupported",
+ "distribution disabled",
+ "distribution api",
+ }
+
+ for _, indicator := range indicators {
+ if strings.Contains(errLower, indicator) {
+ return true
+ }
+ }
+
+ return false
+}
+
+func FetchDigest(ctx context.Context, registryHost, repository, tag string, credential *Credentials) (string, error) {
+ return FetchDigestWithHTTPClient(ctx, registryHost, repository, tag, credential, nil)
+}
+
+func FetchDigestWithHTTPClient(ctx context.Context, registryHost, repository, tag string, credential *Credentials, httpClient *http.Client) (string, error) {
+ if httpClient == nil {
+ httpClient = NewRegistryHTTPClient()
+ }
+
+ requestCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+
+ authHeader := ""
+ if credential != nil && strings.TrimSpace(credential.Username) != "" && strings.TrimSpace(credential.Token) != "" {
+ authHeader = basicAuthHeaderInternal(credential.Username, credential.Token)
+ }
+
+ resp, err := manifestRequestInternal(requestCtx, httpClient, registryHost, repository, tag, authHeader)
+ if err != nil {
+ return "", err
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ if resp.StatusCode == http.StatusUnauthorized {
+ challenge := resp.Header.Get("WWW-Authenticate")
+ if challenge == "" {
+ return "", fmt.Errorf("manifest request failed with status: %d", resp.StatusCode)
+ }
+ return fetchWithTokenAuthInternal(requestCtx, httpClient, registryHost, repository, tag, challenge, credential)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("manifest request failed with status: %d", resp.StatusCode)
+ }
+
+ digest := extractDigestFromHeadersInternal(resp.Header)
+ if digest == "" {
+ return "", fmt.Errorf("no digest header found in response")
+ }
+
+ return digest, nil
+}
+
+func fetchWithTokenAuthInternal(ctx context.Context, httpClient *http.Client, registryHost, repository, tag, challenge string, credential *Credentials) (string, error) {
+ realm, service := parseWWWAuthInternal(challenge)
+ if realm == "" {
+ return "", fmt.Errorf("no auth realm found")
+ }
+
+ token, err := fetchRegistryTokenInternal(ctx, httpClient, realm, service, repository, credential)
+ if err != nil {
+ return "", err
+ }
+
+ resp, err := manifestRequestInternal(ctx, httpClient, registryHost, repository, tag, token)
+ if err != nil {
+ return "", err
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ if resp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("authenticated manifest request failed with status: %d", resp.StatusCode)
+ }
+
+ digest := extractDigestFromHeadersInternal(resp.Header)
+ if digest == "" {
+ return "", fmt.Errorf("no digest header found in authenticated response")
+ }
+
+ return digest, nil
+}
+
+func fetchRegistryTokenInternal(ctx context.Context, httpClient *http.Client, authURL, service, repository string, credential *Credentials) (string, error) {
+ parsed, err := url.Parse(authURL)
+ if err != nil {
+ return "", fmt.Errorf("invalid auth url: %w", err)
+ }
+
+ query := parsed.Query()
+ if query.Get("service") == "" {
+ if strings.TrimSpace(service) != "" {
+ query.Set("service", strings.TrimSpace(service))
+ } else {
+ query.Set("service", serviceNameFromAuthURLInternal(authURL))
+ }
+ }
+ query.Add("scope", fmt.Sprintf("repository:%s:pull", repository))
+ parsed.RawQuery = query.Encode()
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil)
+ if err != nil {
+ return "", fmt.Errorf("create token request: %w", err)
+ }
+ if credential != nil && strings.TrimSpace(credential.Username) != "" && strings.TrimSpace(credential.Token) != "" {
+ req.SetBasicAuth(strings.TrimSpace(credential.Username), strings.TrimSpace(credential.Token))
+ }
+
+ resp, err := httpClient.Do(req) //nolint:gosec // authURL comes from the registry challenge for the current image
+ if err != nil {
+ return "", fmt.Errorf("token request failed: %w", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ if resp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("token request failed with status: %d", resp.StatusCode)
+ }
+
+ var tokenResponse struct {
+ Token string `json:"token"`
+ Legacy string `json:"access_token"`
+ }
+ if err := json.NewDecoder(resp.Body).Decode(&tokenResponse); err != nil {
+ return "", fmt.Errorf("decode token response: %w", err)
+ }
+
+ token := strings.TrimSpace(tokenResponse.Token)
+ if token == "" {
+ token = strings.TrimSpace(tokenResponse.Legacy)
+ }
+ if token == "" {
+ return "", fmt.Errorf("no token in response")
+ }
+
+ return token, nil
+}
+
+func manifestRequestInternal(ctx context.Context, httpClient *http.Client, registryHost, repository, tag, authHeader string) (*http.Response, error) {
+ manifestURL := fmt.Sprintf("%s/v2/%s/manifests/%s", registryBaseURLInternal(registryHost), repository, tag)
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodHead, manifestURL, nil)
+ if err != nil {
+ return nil, fmt.Errorf("create manifest request: %w", err)
+ }
+ addManifestRequestHeadersInternal(req, authHeader)
+
+ resp, err := httpClient.Do(req) //nolint:gosec // manifestURL is derived from the normalized image reference
+ if err != nil {
+ return nil, fmt.Errorf("manifest request failed: %w", err)
+ }
+
+ switch resp.StatusCode {
+ case http.StatusMethodNotAllowed:
+ // Retry with GET only when the registry rejects HEAD as an unsupported method.
+ _ = resp.Body.Close()
+
+ getReq, err := http.NewRequestWithContext(ctx, http.MethodGet, manifestURL, nil)
+ if err != nil {
+ return nil, fmt.Errorf("create manifest fallback request: %w", err)
+ }
+ addManifestRequestHeadersInternal(getReq, authHeader)
+
+ getResp, err := httpClient.Do(getReq) //nolint:gosec // manifestURL is derived from the normalized image reference
+ if err != nil {
+ return nil, fmt.Errorf("manifest fallback request failed: %w", err)
+ }
+
+ return getResp, nil
+ default:
+ return resp, nil
+ }
+}
+
+// NewRegistryHTTPClient returns the shared transport configuration used for
+// direct registry digest lookups.
+func NewRegistryHTTPClient() *http.Client {
+ var transport *http.Transport
+ if defaultTransport, ok := http.DefaultTransport.(*http.Transport); ok {
+ transport = defaultTransport.Clone()
+ } else {
+ transport = &http.Transport{}
+ }
+ transport.Proxy = http.ProxyFromEnvironment
+
+ return &http.Client{Transport: transport}
+}
+
+func registryBaseURLInternal(registryHost string) string {
+ trimmed := strings.TrimSpace(registryHost)
+ if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") {
+ if parsed, err := url.Parse(trimmed); err == nil && parsed.Host != "" && normalizeRegistryForComparisonInternal(parsed.Host) == "docker.io" {
+ return "https://" + defaultRegistryHost
+ }
+ return strings.TrimSuffix(trimmed, "/")
+ }
+
+ normalizedHost := normalizeRegistryForComparisonInternal(trimmed)
+ if normalizedHost == "docker.io" {
+ return "https://" + defaultRegistryHost
+ }
+
+ return "https://" + normalizedHost
+}
+
+func addManifestRequestHeadersInternal(req *http.Request, authHeader string) {
+ req.Header.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json")
+ req.Header.Set("User-Agent", "Arcane")
+ if strings.TrimSpace(authHeader) != "" {
+ req.Header.Set("Authorization", buildAuthHeaderInternal(authHeader))
+ }
+}
+
+func buildAuthHeaderInternal(authHeader string) string {
+ trimmed := strings.TrimSpace(authHeader)
+ if trimmed == "" {
+ return ""
+ }
+
+ lower := strings.ToLower(trimmed)
+ if strings.HasPrefix(lower, "bearer ") || strings.HasPrefix(lower, "basic ") {
+ return trimmed
+ }
+
+ return "Bearer " + trimmed
+}
+
+func basicAuthHeaderInternal(username, token string) string {
+ raw := strings.TrimSpace(username) + ":" + strings.TrimSpace(token)
+ return "Basic " + base64.StdEncoding.EncodeToString([]byte(raw))
+}
+
+func extractDigestFromHeadersInternal(headers http.Header) string {
+ if digest := headers.Get("Docker-Content-Digest"); digest != "" {
+ return digest
+ }
+
+ etag := strings.Trim(headers.Get("ETag"), `"`)
+ if strings.HasPrefix(etag, "sha256:") {
+ return etag
+ }
+
+ return ""
+}
+
+func parseWWWAuthInternal(header string) (string, string) {
+ lower := strings.ToLower(header)
+ if !strings.HasPrefix(lower, "bearer ") {
+ return "", ""
+ }
+
+ _, after, ok := strings.Cut(header, " ")
+ if !ok {
+ return "", ""
+ }
+
+ var realm string
+ var service string
+ for _, part := range splitBearerDirectivesInternal(after) {
+ part = strings.TrimSpace(part)
+ lowerPart := strings.ToLower(part)
+
+ switch {
+ case strings.HasPrefix(lowerPart, "realm="):
+ realm = strings.Trim(part[len("realm="):], `"`)
+ case strings.HasPrefix(lowerPart, "service="):
+ service = strings.Trim(part[len("service="):], `"`)
+ }
+ }
+
+ return realm, service
+}
+
+func splitBearerDirectivesInternal(value string) []string {
+ var parts []string
+ var current strings.Builder
+ inQuote := false
+
+ for _, r := range value {
+ switch {
+ case r == '"':
+ inQuote = !inQuote
+ current.WriteRune(r)
+ case r == ',' && !inQuote:
+ parts = append(parts, current.String())
+ current.Reset()
+ default:
+ current.WriteRune(r)
+ }
+ }
+
+ if current.Len() > 0 {
+ parts = append(parts, current.String())
+ }
+
+ return parts
+}
+
+func serviceNameFromAuthURLInternal(authURL string) string {
+ if strings.Contains(authURL, "auth.docker.io") {
+ return "registry.docker.io"
+ }
+
+ trimmed := strings.TrimPrefix(authURL, "https://")
+ trimmed = strings.TrimPrefix(trimmed, "http://")
+ host, _, _ := strings.Cut(trimmed, "/")
+ if host == "" {
+ return "registry"
+ }
+
+ return host
+}
+
+func normalizeRegistryForComparisonInternal(raw string) string {
+ value := strings.TrimSpace(strings.ToLower(raw))
+ value = strings.TrimPrefix(value, "https://")
+ value = strings.TrimPrefix(value, "http://")
+ value = strings.TrimSuffix(value, "/")
+
+ if slash := strings.Index(value, "/"); slash != -1 {
+ value = value[:slash]
+ }
+
+ switch value {
+ case "docker.io", "registry-1.docker.io", "index.docker.io":
+ return "docker.io"
+ default:
+ return value
+ }
+}
diff --git a/backend/pkg/utils/distribution/distribution_test.go b/backend/pkg/utils/distribution/distribution_test.go
new file mode 100644
index 000000000..7d6a72109
--- /dev/null
+++ b/backend/pkg/utils/distribution/distribution_test.go
@@ -0,0 +1,97 @@
+package distribution
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestFetchDigestWithHTTPClient_FallsBackToGetOnMethodNotAllowed(t *testing.T) {
+ var headCalls int
+ var getCalls int
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case http.MethodHead:
+ headCalls++
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ case http.MethodGet:
+ getCalls++
+ w.Header().Set("Docker-Content-Digest", "sha256:method-not-allowed")
+ w.WriteHeader(http.StatusOK)
+ default:
+ t.Fatalf("unexpected method %s", r.Method)
+ }
+ }))
+ defer server.Close()
+
+ digest, err := FetchDigestWithHTTPClient(
+ context.Background(),
+ server.URL,
+ "team/app",
+ "1.2.3",
+ nil,
+ server.Client(),
+ )
+ require.NoError(t, err)
+ assert.Equal(t, "sha256:method-not-allowed", digest)
+ assert.Equal(t, 1, headCalls)
+ assert.Equal(t, 1, getCalls)
+}
+
+func TestFetchDigestWithHTTPClient_DoesNotFallbackToGetOnResourceErrors(t *testing.T) {
+ testCases := []struct {
+ name string
+ status int
+ }{
+ {name: "not found", status: http.StatusNotFound},
+ {name: "forbidden", status: http.StatusForbidden},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ var headCalls int
+ var getCalls int
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case http.MethodHead:
+ headCalls++
+ w.WriteHeader(tc.status)
+ case http.MethodGet:
+ getCalls++
+ w.Header().Set("Docker-Content-Digest", "sha256:unexpected-get")
+ w.WriteHeader(http.StatusOK)
+ default:
+ t.Fatalf("unexpected method %s", r.Method)
+ }
+ }))
+ defer server.Close()
+
+ digest, err := FetchDigestWithHTTPClient(
+ context.Background(),
+ server.URL,
+ "team/app",
+ "1.2.3",
+ nil,
+ server.Client(),
+ )
+ require.Error(t, err)
+ assert.Empty(t, digest)
+ assert.Equal(t, fmt.Sprintf("manifest request failed with status: %d", tc.status), err.Error())
+ assert.Equal(t, 1, headCalls)
+ assert.Equal(t, 0, getCalls)
+ })
+ }
+}
+
+func TestParseWWWAuthInternal_AllowsCommasInsideQuotedRealm(t *testing.T) {
+ realm, service := parseWWWAuthInternal(`Bearer realm="https://auth.example.com/token?a=1,b=2",service="registry.example.com"`)
+ assert.Equal(t, "https://auth.example.com/token?a=1,b=2", realm)
+ assert.Equal(t, "registry.example.com", service)
+}