Skip to content

Modern Docker Management, Designed for Everyone

fix: refactor image digest inspection logic (#3826)

jdrouhardd256fc2787ef
diff --git a/backend/internal/registry/service.go b/backend/internal/registry/service.go
index 75a7d2299..2048cd051 100644
--- a/backend/internal/registry/service.go
+++ b/backend/internal/registry/service.go
@@ -913,150 +913,66 @@ func (s *ContainerRegistryService) InspectImageDigest(ctx context.Context, image
return lastResult, nil
}
+type digestFetchFunc func(ctx context.Context, credential *resolvedRegistryCredential) (string, error)
+
func (s *ContainerRegistryService) inspectImageDigestViaDaemonInternal(ctx context.Context, normalizedRef, registryHost string, externalCreds []containerregistry.Credential) (*containerregistry.DigestResult, error) {
dockerClient, err := s.getDockerClientInternal(ctx)
if err != nil {
return nil, err
}
- // Use stored credentials up front for any registry that has them. Anonymous
- // requests share a per-registry quota with every other unauthenticated client,
- // so an anonymous first attempt is rate limited for reasons unrelated to us.
- if credentials, credErr := s.getMatchingRegistryCredentialsInternal(ctx, registryHost, externalCreds); credErr == nil && len(credentials) > 0 {
- result, credentialErr := s.inspectImageDigestWithCredentialsInternal(ctx, dockerClient, normalizedRef, registryHost, credentials, nil)
- // Docker Hub anonymous quotas are too small to be worth a retry; elsewhere a
- // stale credential must not break public images that resolve anonymously.
- if credentialErr == nil || isDockerHubRegistryInternal(registryHost) {
- return result, credentialErr
- }
- slog.DebugContext(ctx, "credentialed distribution inspect failed, retrying anonymously",
- "registry", registryHost,
- "imageRef", normalizedRef,
- "error", credentialErr.Error())
- }
+ return s.inspectImageDigestWithCredentialsInternal(ctx, registryHost, "distribution inspect of "+normalizedRef, externalCreds,
+ func(ctx context.Context, credential *resolvedRegistryCredential) (string, error) {
+ return s.fetchDigestFromDaemonInternal(ctx, dockerClient, registryHost, normalizedRef, credential)
+ })
+}
- inspectResult, err := dockerClient.DistributionInspect(ctx, normalizedRef, client.DistributionInspectOptions{})
- if err == nil {
- digestValue, normalizeErr := digest.Normalize(inspectResult.Descriptor.Digest.String())
- if normalizeErr != nil {
- return nil, errors.WrapIff(normalizeErr, "distribution inspect returned invalid digest for %s", normalizedRef)
- }
- return &containerregistry.DigestResult{
- Digest: digestValue,
- AuthMethod: "anonymous",
- AuthRegistry: registryHost,
- }, nil
- }
- if !isUnauthorizedRegistryErrorInternal(err) {
- return &containerregistry.DigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}, errors.
- WrapIff(err, "distribution inspect failed for %s", normalizedRef)
- }
+func (s *ContainerRegistryService) inspectImageDigestViaRegistryInternal(ctx context.Context, registryHost, repository, tag string, externalCreds []containerregistry.Credential) (*containerregistry.DigestResult, error) {
+ return s.inspectImageDigestWithCredentialsInternal(ctx, registryHost, "registry manifest inspect of "+registryHost+"/"+repository+":"+tag, externalCreds,
+ func(ctx context.Context, credential *resolvedRegistryCredential) (string, error) {
+ return s.fetchDigestFromRegistryInternal(ctx, registryHost, repository, tag, credential)
+ })
+}
+// Stored credentials go first: anonymous requests share a per-registry quota with every other unauthenticated client.
+func (s *ContainerRegistryService) inspectImageDigestWithCredentialsInternal(ctx context.Context, registryHost, operation string, externalCreds []containerregistry.Credential, fetch digestFetchFunc) (*containerregistry.DigestResult, error) {
credentials, credErr := s.getMatchingRegistryCredentialsInternal(ctx, registryHost, externalCreds)
- if credErr != nil {
- return &containerregistry.DigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}, errors.
- WrapIf(stderrors.Join(err, credErr), "distribution inspect: anonymous access unauthorized; credential lookup failed")
- }
- return s.inspectImageDigestWithCredentialsInternal(ctx, dockerClient, normalizedRef, registryHost, credentials, err)
-}
-
-func (s *ContainerRegistryService) inspectImageDigestWithCredentialsInternal(ctx context.Context, dockerClient RegistryDaemonClient, normalizedRef, registryHost string, credentials []resolvedRegistryCredential, previousErr error) (*containerregistry.DigestResult, error) {
- lastErr := previousErr
- var lastCred resolvedRegistryCredential
+ var lastErr error
+ var lastResult *containerregistry.DigestResult
for _, credential := range credentials {
- lastCred = credential
- authHeader, encodeErr := utilsregistry.EncodeAuthHeader(credential.Username, credential.Token, credential.ServerAddress)
- if encodeErr != nil {
- return nil, errors.WrapIff(encodeErr, "encode registry auth header for %s", registryHost)
- }
-
- inspectResult, err := dockerClient.DistributionInspect(ctx, normalizedRef, client.DistributionInspectOptions{
- EncodedRegistryAuth: authHeader,
- })
+ lastResult = &containerregistry.DigestResult{AuthMethod: "credential", AuthUsername: credential.Username, AuthRegistry: registryHost, UsedCredential: true}
+ digestValue, err := fetch(ctx, &credential)
if err == nil {
- digestValue, normalizeErr := digest.Normalize(inspectResult.Descriptor.Digest.String())
- if normalizeErr != nil {
- return nil, errors.WrapIff(normalizeErr, "distribution inspect returned invalid digest for %s", normalizedRef)
- }
- return &containerregistry.DigestResult{
- Digest: digestValue,
- AuthMethod: "credential",
- AuthUsername: credential.Username,
- AuthRegistry: registryHost,
- UsedCredential: true,
- }, nil
+ lastResult.Digest = digestValue
+ return lastResult, nil
}
lastErr = err
if !isUnauthorizedRegistryErrorInternal(err) {
- return &containerregistry.DigestResult{
- AuthMethod: "credential",
- AuthUsername: credential.Username,
- AuthRegistry: registryHost,
- UsedCredential: true,
- }, errors.WrapIff(err, "distribution inspect failed for %s with credentials", normalizedRef)
+ return lastResult, errors.WrapIff(err, "%s failed with credentials", operation)
}
}
-
- partial := &containerregistry.DigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}
- if lastCred.Username != "" {
- partial.AuthMethod = "credential"
- partial.AuthUsername = lastCred.Username
- partial.UsedCredential = true
- }
- if lastErr == nil {
- return partial, errors.Errorf("distribution inspect failed for %s: no credentials available", normalizedRef)
+ if lastErr != nil {
+ // Docker Hub anonymous quotas are too small to be worth a retry; elsewhere a stale credential must not break public images.
+ if isDockerHubRegistryInternal(registryHost) {
+ return lastResult, errors.WrapIff(lastErr, "%s failed", operation)
+ }
+ slog.DebugContext(ctx, "credentialed digest lookup failed, retrying anonymously",
+ "registry", registryHost,
+ "operation", operation,
+ "error", lastErr.Error())
}
- return partial, errors.WrapIff(lastErr, "distribution inspect failed for %s", normalizedRef)
-}
-func (s *ContainerRegistryService) inspectImageDigestViaRegistryInternal(ctx context.Context, registryHost, repository, tag string, externalCreds []containerregistry.Credential) (*containerregistry.DigestResult, error) {
- digestValue, err := s.fetchDigestFromRegistryInternal(ctx, registryHost, repository, tag, nil)
+ result := &containerregistry.DigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}
+ digestValue, err := fetch(ctx, nil)
if err == nil {
- return &containerregistry.DigestResult{
- Digest: digestValue,
- AuthMethod: "anonymous",
- AuthRegistry: registryHost,
- }, nil
+ result.Digest = digestValue
+ return result, nil
}
- if !isUnauthorizedRegistryErrorInternal(err) {
- return &containerregistry.DigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}, errors.
- WrapIff(err, "registry manifest inspect failed for %s/%s:%s", registryHost, repository, tag)
+ if credErr != nil && isUnauthorizedRegistryErrorInternal(err) {
+ return result, errors.WrapIff(stderrors.Join(err, credErr), "%s: anonymous access unauthorized; credential lookup failed", operation)
}
-
- credentials, credErr := s.getMatchingRegistryCredentialsInternal(ctx, registryHost, externalCreds)
- if credErr != nil {
- return &containerregistry.DigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}, errors.
- WrapIf(stderrors.Join(err, credErr), "registry manifest inspect: anonymous access unauthorized; credential lookup failed")
- }
-
- lastErr := err
- var lastCred resolvedRegistryCredential
- for _, credential := range credentials {
- lastCred = credential
-
- digestValue, err = s.fetchDigestFromRegistryInternal(ctx, registryHost, repository, tag, &credential)
- if err == nil {
- return &containerregistry.DigestResult{
- Digest: digestValue,
- AuthMethod: "credential",
- AuthUsername: credential.Username,
- AuthRegistry: registryHost,
- UsedCredential: true,
- }, nil
- }
-
- lastErr = err
- }
-
- partial := &containerregistry.DigestResult{AuthMethod: "anonymous", AuthRegistry: registryHost}
- if lastCred.Username != "" {
- partial.AuthMethod = "credential"
- partial.AuthUsername = lastCred.Username
- partial.UsedCredential = true
- }
-
- return partial, errors.WrapIff(lastErr, "registry manifest inspect failed for %s/%s:%s", registryHost, repository, tag)
+ return result, errors.WrapIff(err, "%s failed", operation)
}
func (s *ContainerRegistryService) getDockerClientInternal(ctx context.Context) (RegistryDaemonClient, error) {
@@ -1478,6 +1394,27 @@ func isDistributionFallbackEligibleInternal(err error) bool {
strings.Contains(errLower, "i/o timeout")
}
+func (s *ContainerRegistryService) fetchDigestFromDaemonInternal(ctx context.Context, dockerClient RegistryDaemonClient, registryHost, normalizedRef string, credential *resolvedRegistryCredential) (string, error) {
+ var inspectOptions client.DistributionInspectOptions
+ if credential != nil {
+ authHeader, err := utilsregistry.EncodeAuthHeader(credential.Username, credential.Token, credential.ServerAddress)
+ if err != nil {
+ return "", errors.WrapIff(err, "encode registry auth header for %s", registryHost)
+ }
+ inspectOptions.EncodedRegistryAuth = authHeader
+ }
+
+ inspectResult, err := dockerClient.DistributionInspect(ctx, normalizedRef, inspectOptions)
+ if err != nil {
+ return "", err
+ }
+ digestValue, err := digest.Normalize(inspectResult.Descriptor.Digest.String())
+ if err != nil {
+ return "", errors.WrapIff(err, "distribution inspect returned invalid digest for %s", normalizedRef)
+ }
+ return digestValue, nil
+}
+
func (s *ContainerRegistryService) fetchDigestFromRegistryInternal(ctx context.Context, registryHost, repository, tag string, credential *resolvedRegistryCredential) (string, error) {
var distributionCredential *registry.Credentials
if credential != nil {
diff --git a/backend/internal/registry/service_test.go b/backend/internal/registry/service_test.go
index 878691f3c..b83b3518f 100644
--- a/backend/internal/registry/service_test.go
+++ b/backend/internal/registry/service_test.go
@@ -2,7 +2,9 @@ package registry
import (
"context"
+ "encoding/base64"
"encoding/json"
+ "io"
"net/http"
"net/http/httptest"
"net/url"
@@ -937,6 +939,71 @@ func TestContainerRegistryService_InspectImageDigest_FallsBackToAnonymousWhenSto
assert.False(t, result.UsedCredential)
}
+// A rate-limited credential must not fall back to anonymous: both paths share the per-registry quota.
+func TestContainerRegistryService_InspectImageDigest_DoesNotFallBackToAnonymousOnCredentialRateLimit(t *testing.T) {
+ db := setupContainerRegistryTestDBInternal(t)
+ createTestPullRegistryInternal(t, db, "ghcr.io", "ghcr-user", "ghcr-token")
+
+ var anonymousCalls int
+ svc := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ if options.EncodedRegistryAuth == "" {
+ anonymousCalls++
+ }
+ return client.DistributionInspectResult{}, errors.New(
+ "Error response from daemon: toomanyrequests: retry-after: 1.246809ms, allowed: 44000/minute")
+ },
+ }, nil
+ }, nil)
+
+ result, err := svc.InspectImageDigest(context.Background(), "ghcr.io/getarcaneapp/agent:latest", nil)
+ require.Error(t, err)
+ assert.Zero(t, anonymousCalls, "must not burn anonymous quota when the credential itself is rate limited")
+ require.NotNil(t, result)
+ assert.Equal(t, "credential", result.AuthMethod)
+ assert.True(t, result.UsedCredential)
+}
+
+// The direct-HTTP fallback must not spend an anonymous request when a stored credential resolves the digest.
+func TestContainerRegistryService_InspectImageDigest_FallbackUsesStoredCredentialsBeforeAnonymous(t *testing.T) {
+ db := setupContainerRegistryTestDBInternal(t)
+ createTestPullRegistryInternal(t, db, "ghcr.io", "ghcr-user", "ghcr-token")
+ wantDigest := digest.FromString("fallback-stored-credentials").String()
+
+ var anonymousCalls int
+ svc := NewContainerRegistryService(db, func(context.Context) (RegistryDaemonClient, error) {
+ return &fakeRegistryDaemonClient{
+ distributionInspectFn: func(ctx context.Context, imageRef string, options client.DistributionInspectOptions) (client.DistributionInspectResult, error) {
+ return client.DistributionInspectResult{}, errors.New("Error response from daemon: Not Found")
+ },
+ }, nil
+ }, nil)
+ svc.distributionHTTPClient = &http.Client{
+ Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ authorization := req.Header.Get("Authorization")
+ if authorization == "" {
+ anonymousCalls++
+ return nil, errors.New("unexpected anonymous registry request")
+ }
+ assert.Equal(t, "Basic "+base64.StdEncoding.EncodeToString([]byte("ghcr-user:ghcr-token")), authorization)
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: http.Header{"Docker-Content-Digest": []string{wantDigest}},
+ Body: io.NopCloser(strings.NewReader("")),
+ }, nil
+ }),
+ }
+
+ result, err := svc.InspectImageDigest(context.Background(), "ghcr.io/getarcaneapp/agent:latest", nil)
+ require.NoError(t, err)
+ assert.Zero(t, anonymousCalls)
+ assert.Equal(t, wantDigest, result.Digest)
+ assert.Equal(t, "credential", result.AuthMethod)
+ assert.Equal(t, "ghcr-user", result.AuthUsername)
+ assert.True(t, result.UsedCredential)
+}
+
func TestContainerRegistryService_InspectImageDigest_FallsBackWhenDistributionNotFound(t *testing.T) {
wantDigest := digest.FromString("fallback-not-found").String()
@@ -1028,6 +1095,9 @@ func TestContainerRegistryService_InspectImageDigest_RetriesStoredCredentialsAft
case 2:
w.WriteHeader(http.StatusForbidden)
case 3:
+ w.Header().Set("WWW-Authenticate", `Bearer realm="`+tokenURL+`",service="registry.example.com"`)
+ w.WriteHeader(http.StatusUnauthorized)
+ case 4:
w.Header().Set("Docker-Content-Digest", wantDigest)
w.WriteHeader(http.StatusOK)
default:
@@ -1083,13 +1153,14 @@ func TestContainerRegistryService_InspectImageDigest_RetriesStoredCredentialsAft
result, err := svc.InspectImageDigest(context.Background(), serverURL.Host+"/team/app:1.2.3", nil)
require.NoError(t, err)
assert.Equal(t, wantDigest, result.Digest)
- assert.Equal(t, "credential", result.AuthMethod)
- assert.Equal(t, "stored-user", result.AuthUsername)
- assert.True(t, result.UsedCredential)
- require.Len(t, authHeaders, 3)
- assert.Empty(t, authHeaders[0])
- assert.Equal(t, "Bearer anonymous-token", authHeaders[1])
- assert.Equal(t, "Basic c3RvcmVkLXVzZXI6c3RvcmVkLXRva2Vu", authHeaders[2])
+ // Stored credentials go first; only after rejection does the lookup retry anonymously.
+ require.Len(t, authHeaders, 4)
+ assert.Equal(t, "Basic c3RvcmVkLXVzZXI6c3RvcmVkLXRva2Vu", authHeaders[0])
+ assert.Equal(t, "Bearer credential-token", authHeaders[1])
+ assert.Empty(t, authHeaders[2])
+ assert.Equal(t, "Bearer anonymous-token", authHeaders[3])
+ assert.Equal(t, "anonymous", result.AuthMethod)
+ assert.False(t, result.UsedCredential)
}
func TestContainerRegistryService_InspectImageDigest_DoesNotFallbackOnTLSFailure(t *testing.T) {