Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

36.8 KB
Raw Blame
package dockeragent

import (
	"context"
	"crypto/sha256"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"sync"
	"time"

	"github.com/rs/zerolog"
)

// RegistryChecker handles container image digest lookups against registries.
type RegistryChecker struct {
	httpClient *http.Client
	cache      *digestCache
	// credentials optionally resolves host Docker credentials so private
	// registries can be checked; nil keeps every lookup anonymous.
	credentials registryCredentialSource
	localImages localImageMemo
	logger      zerolog.Logger
	mu          sync.RWMutex

	// Configuration
	enabled       bool
	checkInterval time.Duration
	lastFullCheck time.Time
}

// localImageDigests is the local side of an update comparison: the image's
// RepoDigests and the platform used to resolve a manifest list.
type localImageDigests struct {
	digests []string
	arch    string
	os      string
	variant string
}

// localImageMemo keeps the last successful local digest lookup per image ID,
// the fallback when an inspect fails. It holds only images in use by the
// collected containers.
type localImageMemo struct {
	entries map[string]localImageDigests
	mu      sync.Mutex
}

func (r *RegistryChecker) rememberLocalImage(imageID string, local localImageDigests) {
	if r == nil || imageID == "" {
		return
	}
	r.localImages.mu.Lock()
	defer r.localImages.mu.Unlock()
	if r.localImages.entries == nil {
		r.localImages.entries = make(map[string]localImageDigests)
	}
	local.digests = append([]string(nil), local.digests...)
	r.localImages.entries[imageID] = local
}

func (r *RegistryChecker) lastLocalImage(imageID string) (localImageDigests, bool) {
	if r == nil || imageID == "" {
		return localImageDigests{}, false
	}
	r.localImages.mu.Lock()
	defer r.localImages.mu.Unlock()
	local, ok := r.localImages.entries[imageID]
	if !ok {
		return localImageDigests{}, false
	}
	local.digests = append([]string(nil), local.digests...)
	return local, true
}

func (r *RegistryChecker) pruneLocalImages(active map[string]struct{}) {
	if r == nil {
		return
	}
	r.localImages.mu.Lock()
	defer r.localImages.mu.Unlock()
	for imageID := range r.localImages.entries {
		if _, ok := active[imageID]; !ok {
			delete(r.localImages.entries, imageID)
		}
	}
}

// digestCache provides thread-safe caching of digest lookups.
type digestCache struct {
	entries map[string]cacheEntry
	mu      sync.RWMutex
	// generation counts ForceCheck clears. A lookup reads it with its cache
	// decision and stores its outcome only if no clear happened since, so a
	// lookup begun before a manual check cannot refill the cleared cache.
	generation uint64
}

type cacheEntry struct {
	latestDigest      string
	comparisonDigests string
	expiresAt         time.Time
	err               string // cached error message
	// Successes remember local images already compared with this tag. Mixed
	// old/new replicas must share the refreshed result, not invalidate it in turn.
	observedLocalDigests string
	refreshAfter         time.Time
	refreshing           bool
}

const (
	// DefaultCacheTTL is the default time-to-live for cached digests.
	defaultCacheTTL = 6 * time.Hour
	// A newly observed local image can supersede a success before its TTL.
	// Bound rapid replacements per reference/platform, not per container.
	localImageRefreshInterval = time.Minute
	maxObservedLocalDigests   = 64
	// ErrorCacheTTL is the TTL for caching errors (shorter to allow retry).
	errorCacheTTL = 15 * time.Minute
	// RateLimitCacheTTL is the TTL for rate-limited lookups. Retrying these on
	// the short error TTL keeps hammering a registry whose allowance is
	// already exhausted (every refused HEAD still counts against it), which
	// can hold a strict registry's limit tripped indefinitely. Back off for
	// an hour instead so the allowance can actually recover.
	rateLimitCacheTTL = 1 * time.Hour
	// DefaultCheckInterval is how often to check for updates.
	defaultCheckInterval = 6 * time.Hour
)

const imageChangeRefreshPending = "local image changed; registry refresh pending"

// ImageUpdateResult contains the result of an image update check.
type ImageUpdateResult struct {
	Image           string    `json:"image"`
	CurrentDigest   string    `json:"currentDigest"`
	LatestDigest    string    `json:"latestDigest"`
	UpdateAvailable bool      `json:"updateAvailable"`
	CheckedAt       time.Time `json:"checkedAt"`
	Error           string    `json:"error,omitempty"`
}

func isPulseManagedImageReference(image string) bool {
	normalized := strings.ToLower(strings.TrimSpace(image))
	if host, rest, explicit := strings.Cut(normalized, "/"); explicit && isDockerHubRegistry(host) {
		normalized = rest
	}
	for _, repository := range []string{
		"rcourtman/pulse",
		"license.pulserelay.pro/pulse-pro",
		"registry.pulserelay.pro/pulse/pulse-pro",
	} {
		if normalized == repository ||
			strings.HasPrefix(normalized, repository+":") ||
			strings.HasPrefix(normalized, repository+"@") {
			return true
		}
	}
	return false
}

// Hub reference and credential aliases name the same registry, but manifests
// and pull tokens must use the v2 API rather than Docker Hub's web/index hosts.
// A port or a lookalike domain is a distinct registry and must not be rewritten.
func isDockerHubRegistry(host string) bool {
	switch strings.ToLower(host) {
	case "docker.io", "index.docker.io", "registry.docker.io", "registry-1.docker.io":
		return true
	}
	return false
}

// NewRegistryChecker creates a new registry checker for the Docker / Podman module.
func NewRegistryChecker(logger zerolog.Logger) *RegistryChecker {
	return newRegistryCheckerWithConfig(logger, true)
}

// newRegistryCheckerWithConfig creates a registry checker with the enabled state set.
func newRegistryCheckerWithConfig(logger zerolog.Logger, enabled bool) *RegistryChecker {
	return &RegistryChecker{
		httpClient: &http.Client{
			Timeout: 30 * time.Second,
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{
					MinVersion: tls.VersionTLS12,
				},
				MaxIdleConns:       10,
				IdleConnTimeout:    90 * time.Second,
				DisableCompression: false,
				DisableKeepAlives:  false,
			},
		},
		cache: &digestCache{
			entries: make(map[string]cacheEntry),
		},
		logger:        logger,
		enabled:       enabled,
		checkInterval: defaultCheckInterval,
	}
}

// SetEnabled enables or disables update checking.
func (r *RegistryChecker) SetEnabled(enabled bool) {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.enabled = enabled
}

// Enabled returns whether update checking is enabled.
func (r *RegistryChecker) Enabled() bool {
	r.mu.RLock()
	defer r.mu.RUnlock()
	return r.enabled
}

// ShouldCheck returns true if enough time has passed since the last full check.
func (r *RegistryChecker) ShouldCheck() bool {
	r.mu.RLock()
	defer r.mu.RUnlock()

	if !r.enabled {
		return false
	}

	return time.Since(r.lastFullCheck) >= r.checkInterval
}

// MarkChecked updates the last check timestamp.
func (r *RegistryChecker) MarkChecked() {
	r.mu.Lock()
	defer r.mu.Unlock()
	r.lastFullCheck = time.Now()
}

// ForceCheck clears the cache and resets the last check timestamp.
func (r *RegistryChecker) ForceCheck() {
	r.mu.Lock()
	r.lastFullCheck = time.Time{}
	r.mu.Unlock()

	r.cache.mu.Lock()
	defer r.cache.mu.Unlock()
	r.cache.entries = make(map[string]cacheEntry)
	r.cache.generation++
}

// proBrokerRegistry is the entitled Pulse Pro registry. Digest checks against
// it require a license credential the agent does not hold, so an anonymous
// HEAD can never succeed and would pin a permanent "authentication required"
// badge on the Pulse Pro container. That container updates through the
// broker's digest-pinned commands instead, so the generic checker stays out.
const proBrokerRegistry = "license.pulserelay.pro"

// localImageUpdateStatus is the update-status sentinel for an image built or
// loaded on the host, which no registry holds. Like "digest-pinned image" it
// is not a failure, and Pulse's update surfaces render it as its own state.
const localImageUpdateStatus = "local image"

// CheckImageUpdate checks if a newer version of the image is available.
func (r *RegistryChecker) CheckImageUpdate(ctx context.Context, image, currentDigest, arch, goos, variant string) *ImageUpdateResult {
	if !r.Enabled() {
		return nil
	}
	if isPulseManagedImageReference(image) {
		return nil
	}

	registry, repository, tag := parseImageReference(image)

	if registry == proBrokerRegistry {
		return nil
	}

	// currentDigest may be a comma-separated set of the local image's
	// RepoDigests; report the primary one while comparing against all of them.
	primaryCurrentDigest := currentDigest
	if idx := strings.Index(currentDigest, ","); idx >= 0 {
		primaryCurrentDigest = strings.TrimSpace(currentDigest[:idx])
	}

	// Skip digest-pinned images (image@sha256:...)
	if registry == "" {
		return &ImageUpdateResult{
			Image:           image,
			CurrentDigest:   primaryCurrentDigest,
			UpdateAvailable: false,
			CheckedAt:       time.Now(),
			Error:           "digest-pinned image",
		}
	}

	// Check cache first
	cacheKey := fmt.Sprintf("%s/%s:%s|%s/%s/%s", registry, repository, tag, arch, goos, variant)
	r.logger.Debug().Str("image", image).Str("cacheKey", cacheKey).Msg("Checking update (internal)")

	cached, refresh, generation := r.cachedForLocalImage(cacheKey, currentDigest)
	if cached != nil && !refresh {
		r.logger.Debug().Str("image", image).Msg("cache hit for update check")
		if cached.err != "" {
			return &ImageUpdateResult{
				Image:           image,
				CurrentDigest:   primaryCurrentDigest,
				UpdateAvailable: false,
				CheckedAt:       time.Now(),
				Error:           cached.err,
			}
		}
		return &ImageUpdateResult{
			Image:           image,
			CurrentDigest:   primaryCurrentDigest,
			LatestDigest:    cached.latestDigest,
			UpdateAvailable: r.digestsDiffer(currentDigest, cached.comparisonDigests),
			CheckedAt:       time.Now(),
		}
	}

	// Fetch latest digest from registry
	latestDigest, headDigest, err := r.fetchDigest(ctx, registry, repository, tag, arch, goos, variant)
	if err != nil {
		// Cache the error to avoid hammering the registry
		if !r.cacheErrorInGeneration(generation, cacheKey, err.Error()) {
			return r.clearedDuringLookup(image)
		}

		r.logger.Debug().
			Str("image", image).
			Str("registry", registry).
			Err(err).
			Msg("Failed to fetch image digest from registry")

		return &ImageUpdateResult{
			Image:           image,
			CurrentDigest:   primaryCurrentDigest,
			UpdateAvailable: false,
			CheckedAt:       time.Now(),
			Error:           err.Error(),
		}
	}

	// Compare against both digest layers. Docker's RepoDigest identifies the
	// tag/index while the resolved digest identifies this host's platform
	// manifest; either one proves that the local image is current.
	comparisonDigests := latestDigest
	if headDigest != latestDigest && headDigest != "" {
		comparisonDigests = latestDigest + "," + headDigest
	}
	publicLatestDigest := latestDigest
	if headDigest != "" {
		publicLatestDigest = headDigest
	}

	// Cache the successful result
	if !r.cacheImageDigestResult(generation, cacheKey, publicLatestDigest, comparisonDigests, currentDigest, cached) {
		return r.clearedDuringLookup(image)
	}

	updateAvailable := r.digestsDiffer(currentDigest, comparisonDigests)

	r.logger.Debug().
		Str("image", image).
		Str("currentDigest", currentDigest).
		Str("latestDigest", latestDigest).
		Str("headDigest", headDigest).
		Str("arch", arch).
		Str("os", goos).
		Str("variant", variant).
		Bool("updateAvailable", updateAvailable).
		Msg("Checked image update")

	return &ImageUpdateResult{
		Image:           image,
		CurrentDigest:   primaryCurrentDigest,
		LatestDigest:    publicLatestDigest,
		UpdateAvailable: updateAvailable,
		CheckedAt:       time.Now(),
	}
}

// digestsDiffer compares two digests, handling format differences.
func (r *RegistryChecker) digestsDiffer(current, latest string) bool {
	if current == "" || latest == "" {
		return false
	}

	// latest may contain multiple comma-separated digests (resolved + head).
	// current may also hold several local RepoDigests for one image (#2110).
	latestSet := make(map[string]struct{})
	for _, l := range strings.Split(latest, ",") {
		if normalized := normalizeComparisonDigest(l); normalized != "" {
			latestSet[normalized] = struct{}{}
		}
	}
	if len(latestSet) == 0 {
		return false
	}

	compared := false
	for _, c := range strings.Split(current, ",") {
		normalized := normalizeComparisonDigest(c)
		if normalized == "" {
			continue
		}
		compared = true
		if _, ok := latestSet[normalized]; ok {
			return false // Match found
		}
	}
	if !compared {
		return false
	}

	return true // No match found
}

// normalizeComparisonDigest lowercases a digest and removes the "sha256:"
// prefix so registry and RepoDigest spellings compare equal.
func normalizeComparisonDigest(digest string) string {
	return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(digest), "sha256:"))
}

// fetchDigest retrieves the digest for an image from the registry.
// Returns the resolved platform-specific digest AND the raw HEAD digest (which might be a manifest list).
func (r *RegistryChecker) fetchDigest(ctx context.Context, registry, repository, tag, arch, goos, variant string) (string, string, error) {
	creds := r.lookupCredentials(ctx, registry)

	// Get auth token if needed
	token, tokenUsedCreds, err := r.getScopedAuthToken(ctx, registry, repository, creds)
	if err != nil {
		return "", "", fmt.Errorf("auth: %w", err)
	}
	authorization := ""
	if token != "" {
		authorization = "Bearer " + token
	}

	// Construct the manifest URL
	manifestURL := fmt.Sprintf("https://%s/v2/%s/manifests/%s", registry, repository, tag)

	resp, err := r.headManifest(ctx, manifestURL, authorization)
	if err != nil {
		return "", "", err
	}

	if resp.StatusCode == http.StatusUnauthorized {
		// Registries without a hardcoded token endpoint (lscr.io, quay.io, ...)
		// advertise it in the WWW-Authenticate challenge. Negotiate a pull
		// token (with host credentials when the store has them) or answer a
		// Basic challenge directly, and retry once.
		challenge := resp.Header.Get("Www-Authenticate")
		if retryAuthorization, ok := r.retryAuthorization(ctx, challenge, repository, token, tokenUsedCreds, creds); ok {
			resp.Body.Close()
			authorization = retryAuthorization
			resp, err = r.headManifest(ctx, manifestURL, authorization)
			if err != nil {
				return "", "", err
			}
		}
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusUnauthorized {
		return "", "", fmt.Errorf("authentication required")
	}
	if resp.StatusCode == http.StatusNotFound {
		return "", "", fmt.Errorf("image not found")
	}
	if resp.StatusCode == http.StatusTooManyRequests {
		return "", "", fmt.Errorf("rate limited")
	}
	if resp.StatusCode >= 400 {
		return "", "", fmt.Errorf("registry error: %d", resp.StatusCode)
	}

	// Get digest from Docker-Content-Digest header
	digest := resp.Header.Get("Docker-Content-Digest")
	if digest == "" {
		// Some registries don't return digest on HEAD, try etag
		digest = resp.Header.Get("Etag")
		if digest != "" {
			// Clean up etag format
			digest = strings.Trim(digest, "\"")
		}
	}

	contentType := resp.Header.Get("Content-Type")
	isManifestList := strings.Contains(contentType, "manifest.list") || strings.Contains(contentType, "image.index")

	if digest == "" {
		if closeErr := resp.Body.Close(); closeErr != nil {
			r.logger.Debug().Err(closeErr).Msg("Failed to close registry HEAD response")
		}
		manifestDigest, manifestContentType, manifestBody, err := r.fetchManifest(ctx, manifestURL, authorization)
		if err != nil {
			return "", "", err
		}
		if len(manifestBody) == 0 {
			return "", "", fmt.Errorf("no digest in response")
		}

		digest = manifestDigest
		if digest == "" {
			digest = fmt.Sprintf("sha256:%x", sha256.Sum256(manifestBody))
		}

		isManifestList = isManifestList ||
			strings.Contains(manifestContentType, "manifest.list") ||
			strings.Contains(manifestContentType, "image.index")
		if isManifestList && arch != "" && goos != "" {
			resolved, resolveErr := resolveManifestListBody(manifestBody, arch, goos, variant)
			return resolved, digest, resolveErr
		}
	}

	// If it's a manifest list and we have arch info, resolve the platform
	// manifest only after establishing the parent digest. Registries that
	// identify an index on HEAD but omit its digest need the GET fallback above
	// to preserve both values.
	if isManifestList && arch != "" && goos != "" {
		resolved, err := r.resolveManifestList(ctx, registry, repository, tag, arch, goos, variant, authorization)
		return resolved, digest, err
	}

	return digest, digest, nil
}

// fetchManifest retrieves a manifest body when a registry accepts HEAD but
// omits the digest headers. The digest of the exact response body is a valid
// fallback under the registry content-addressing contract.
func (r *RegistryChecker) fetchManifest(ctx context.Context, manifestURL, authorization string) (string, string, []byte, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, manifestURL, nil)
	if err != nil {
		return "", "", nil, fmt.Errorf("create manifest request: %w", err)
	}
	r.setManifestRequestHeaders(req, authorization)

	resp, err := r.httpClient.Do(req)
	if err != nil {
		return "", "", nil, fmt.Errorf("manifest request: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusUnauthorized {
		return "", "", nil, fmt.Errorf("authentication required")
	}
	if resp.StatusCode == http.StatusNotFound {
		return "", "", nil, fmt.Errorf("image not found")
	}
	if resp.StatusCode == http.StatusTooManyRequests {
		return "", "", nil, fmt.Errorf("rate limited")
	}
	if resp.StatusCode >= 400 {
		return "", "", nil, fmt.Errorf("registry error: %d", resp.StatusCode)
	}

	body, err := readBodyWithLimit(resp.Body, maxRegistryManifestBodyBytes)
	if err != nil {
		return "", "", nil, fmt.Errorf("read manifest body: %w", err)
	}
	if len(body) == 0 {
		return "", "", nil, fmt.Errorf("no digest in response")
	}

	// Only hash or trust digest metadata for an actual image manifest. A
	// proxy/login/error page with HTTP 200 otherwise becomes a plausible but
	// unrelated sha256 value, shared by every reference receiving that page.
	var manifest struct {
		SchemaVersion int `json:"schemaVersion"`
		Config        struct {
			Digest string `json:"digest"`
		} `json:"config"`
		Manifests []json.RawMessage `json:"manifests"`
	}
	if err := json.Unmarshal(body, &manifest); err != nil || manifest.SchemaVersion != 2 ||
		(manifest.Config.Digest == "" && manifest.Manifests == nil) {
		return "", "", nil, fmt.Errorf("registry returned a non-manifest response")
	}
	digest := strings.Trim(resp.Header.Get("Docker-Content-Digest"), `"`)
	if digest == "" {
		digest = strings.Trim(resp.Header.Get("Etag"), `"`)
	}
	return digest, resp.Header.Get("Content-Type"), body, nil
}

// headManifest issues a manifest HEAD request with the multi-arch Accept set.
func (r *RegistryChecker) headManifest(ctx context.Context, manifestURL, authorization string) (*http.Response, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodHead, manifestURL, nil)
	if err != nil {
		return nil, fmt.Errorf("create request: %w", err)
	}

	r.setManifestRequestHeaders(req, authorization)

	resp, err := r.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("request: %w", err)
	}
	return resp, nil
}

// setManifestRequestHeaders applies the multi-arch Accept set and an optional
// full Authorization header value ("Bearer ..." or "Basic ...").
func (r *RegistryChecker) setManifestRequestHeaders(req *http.Request, authorization string) {
	req.Header.Set("Accept", strings.Join([]string{
		"application/vnd.docker.distribution.manifest.list.v2+json",
		"application/vnd.docker.distribution.manifest.v2+json",
		"application/vnd.oci.image.manifest.v1+json",
		"application/vnd.oci.image.index.v1+json",
	}, ", "))

	if authorization != "" {
		req.Header.Set("Authorization", authorization)
	}
}

// retryAuthorization derives the Authorization value for the single retry
// after an unauthorized manifest HEAD. Bearer challenges are negotiated at
// the advertised token endpoint (anonymously, as before, or with host
// credentials when the store has them); Basic challenges are answered
// directly from host credentials. It returns false when no retry could do
// better than the attempt that already failed.
func (r *RegistryChecker) retryAuthorization(ctx context.Context, challenge, repository, token string, tokenUsedCreds bool, creds *registryCredential) (string, bool) {
	scheme := strings.ToLower(strings.TrimSpace(challenge))
	switch {
	case strings.HasPrefix(scheme, "basic"):
		if creds == nil || creds.IdentityToken {
			return "", false
		}
		return creds.authorizationHeader(), true
	case strings.HasPrefix(scheme, "bearer"):
		if token != "" && (creds == nil || tokenUsedCreds) {
			// The failed attempt already carried our best token; negotiating
			// the same one again cannot succeed.
			return "", false
		}
		negotiated, err := r.tokenFromChallenge(ctx, challenge, repository, creds)
		if err != nil {
			return "", false
		}
		return "Bearer " + negotiated, true
	default:
		return "", false
	}
}

// tokenFromChallenge negotiates a pull token from the token endpoint named in
// a registry's WWW-Authenticate Bearer challenge (Docker registry v2 token
// auth), anonymously or with host credentials.
func (r *RegistryChecker) tokenFromChallenge(ctx context.Context, challenge, repository string, creds *registryCredential) (string, error) {
	params := parseBearerChallenge(challenge)
	realm := params["realm"]
	if realm == "" {
		return "", fmt.Errorf("no bearer challenge")
	}
	scope := params["scope"]
	if scope == "" {
		scope = fmt.Sprintf("repository:%s:pull", repository)
	}
	return r.authorizeToken(ctx, realm, params["service"], scope, creds)
}

// parseBearerChallenge extracts the key="value" parameters from a
// WWW-Authenticate Bearer challenge header.
func parseBearerChallenge(header string) map[string]string {
	params := map[string]string{}
	rest, ok := strings.CutPrefix(strings.TrimSpace(header), "Bearer ")
	if !ok {
		return params
	}
	for _, part := range strings.Split(rest, ",") {
		key, value, found := strings.Cut(strings.TrimSpace(part), "=")
		if !found {
			continue
		}
		params[strings.ToLower(strings.TrimSpace(key))] = strings.Trim(strings.TrimSpace(value), `"`)
	}
	return params
}

// resolveManifestList fetches the manifest list and finds the matching digest for the architecture.
func (r *RegistryChecker) resolveManifestList(ctx context.Context, registry, repository, tag, arch, goos, variant, authorization string) (string, error) {
	manifestURL := fmt.Sprintf("https://%s/v2/%s/manifests/%s", registry, repository, tag)

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, manifestURL, nil)
	if err != nil {
		return "", fmt.Errorf("create list request: %w", err)
	}

	r.setManifestRequestHeaders(req, authorization)

	resp, err := r.httpClient.Do(req)
	if err != nil {
		return "", fmt.Errorf("list request: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("fetch manifest list failed: %d", resp.StatusCode)
	}

	body, err := readBodyWithLimit(resp.Body, maxRegistryManifestBodyBytes)
	if err != nil {
		return "", fmt.Errorf("read list body: %w", err)
	}

	return resolveManifestListBody(body, arch, goos, variant)
}

func resolveManifestListBody(body []byte, arch, goos, variant string) (string, error) {
	var list manifestList
	if err := json.Unmarshal(body, &list); err != nil {
		return "", fmt.Errorf("decode manifest list: %w", err)
	}

	for _, m := range list.Manifests {
		if m.Platform.Architecture == arch && m.Platform.OS == goos {
			if variant != "" && m.Platform.Variant != "" && variant != m.Platform.Variant {
				continue
			}
			return m.Digest, nil
		}
	}

	return "", fmt.Errorf("no matching manifest found for %s/%s in list", goos, arch)
}

type manifestList struct {
	Manifests []manifestDescriptor `json:"manifests"`
}

type manifestDescriptor struct {
	Digest   string           `json:"digest"`
	Platform manifestPlatform `json:"platform"`
}

type manifestPlatform struct {
	Architecture string `json:"architecture"`
	OS           string `json:"os"`
	Variant      string `json:"variant,omitempty"`
}

// getAuthToken retrieves an auth token for the registry.
func (r *RegistryChecker) getAuthToken(ctx context.Context, registry, repository string) (string, error) {
	token, _, err := r.getScopedAuthToken(ctx, registry, repository, r.lookupCredentials(ctx, registry))
	return token, err
}

// getScopedAuthToken negotiates a pull token for registries with hardcoded
// token endpoints. It also reports whether the returned token actually
// carried the supplied credentials, so the 401 retry can tell a fresh
// credentialed attempt apart from replaying one that already failed.
func (r *RegistryChecker) getScopedAuthToken(ctx context.Context, registry, repository string, creds *registryCredential) (string, bool, error) {
	var realm, service string
	switch registry {
	// Docker Hub and ghcr.io require a token even for public images.
	case "registry-1.docker.io":
		realm, service = "https://auth.docker.io/token", "registry.docker.io"
	case "ghcr.io":
		realm, service = "https://ghcr.io/token", "ghcr.io"
	default:
		// For other registries, try anonymous access first.
		return "", false, nil
	}

	if creds != nil {
		token, err := r.authorizeToken(ctx, realm, service, fmt.Sprintf("repository:%s:pull", repository), creds)
		if err == nil {
			return token, true, nil
		}
		// A stale login must not break checks that used to work anonymously.
		r.logger.Debug().Str("registry", registry).Err(err).Msg("Credentialed token negotiation failed; retrying anonymously")
	}

	tokenURL := fmt.Sprintf("%s?service=%s&scope=repository:%s:pull", realm, service, repository)
	token, err := r.fetchAuthToken(ctx, tokenURL)
	return token, false, err
}

// authorizeToken requests a pull token from a token endpoint, attaching Basic
// credentials or exchanging an identity token when host credentials exist.
func (r *RegistryChecker) authorizeToken(ctx context.Context, realm, service, scope string, creds *registryCredential) (string, error) {
	realmURL, err := url.Parse(realm)
	if err != nil || realmURL.Scheme != "https" {
		return "", fmt.Errorf("invalid token realm")
	}

	if creds != nil && creds.IdentityToken {
		return r.exchangeIdentityToken(ctx, realmURL.String(), service, scope, creds)
	}

	query := realmURL.Query()
	if service != "" {
		query.Set("service", service)
	}
	query.Set("scope", scope)
	realmURL.RawQuery = query.Encode()

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, realmURL.String(), nil)
	if err != nil {
		return "", fmt.Errorf("create token request: %w", err)
	}
	if creds != nil {
		req.SetBasicAuth(creds.Username, creds.Secret)
	}
	return r.doTokenRequest(req)
}

// exchangeIdentityToken swaps an OAuth identity token (docker-credential
// helper logins that return Username "<token>", for example Azure ACR) for a
// pull token via the refresh-token grant of the registry token endpoint.
func (r *RegistryChecker) exchangeIdentityToken(ctx context.Context, realm, service, scope string, creds *registryCredential) (string, error) {
	form := url.Values{}
	form.Set("grant_type", "refresh_token")
	form.Set("refresh_token", creds.Secret)
	form.Set("client_id", "pulse-agent")
	if service != "" {
		form.Set("service", service)
	}
	form.Set("scope", scope)

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, realm, strings.NewReader(form.Encode()))
	if err != nil {
		return "", fmt.Errorf("create token request: %w", err)
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return r.doTokenRequest(req)
}

// lookupCredentials resolves host Docker credentials for a registry. It
// returns nil when no source is wired, the registry has no stored login, or
// resolution fails; the check then proceeds anonymously as before.
func (r *RegistryChecker) lookupCredentials(ctx context.Context, registry string) *registryCredential {
	if r.credentials == nil {
		return nil
	}
	return r.credentials.Lookup(ctx, registry)
}

// fetchAuthToken fetches an auth token anonymously from a token endpoint.
func (r *RegistryChecker) fetchAuthToken(ctx context.Context, tokenURL string) (string, error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL, nil)
	if err != nil {
		return "", fmt.Errorf("create token request: %w", err)
	}
	return r.doTokenRequest(req)
}

// doTokenRequest executes a token-endpoint request and decodes the token.
func (r *RegistryChecker) doTokenRequest(req *http.Request) (string, error) {
	resp, err := r.httpClient.Do(req)
	if err != nil {
		return "", fmt.Errorf("send token request: %w", err)
	}
	defer func() {
		if closeErr := resp.Body.Close(); closeErr != nil {
			r.logger.Warn().Err(closeErr).Msg("Failed to close token response body")
		}
	}()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("token request failed: %d", resp.StatusCode)
	}

	body, err := readBodyWithLimit(resp.Body, maxRegistryTokenBodyBytes)
	if err != nil {
		return "", fmt.Errorf("read token response: %w", err)
	}

	var tokenResp struct {
		Token       string `json:"token"`
		AccessToken string `json:"access_token"`
	}
	if err := json.Unmarshal(body, &tokenResp); err != nil {
		return "", fmt.Errorf("decode token response: %w", err)
	}

	if tokenResp.Token != "" {
		return tokenResp.Token, nil
	}
	return tokenResp.AccessToken, nil
}

func (r *RegistryChecker) getCached(key string) *cacheEntry {
	r.cache.mu.RLock()
	defer r.cache.mu.RUnlock()

	entry, ok := r.cache.entries[key]
	if !ok {
		return nil
	}
	if time.Now().After(entry.expiresAt) {
		return nil
	}
	return &entry
}

// cachedForLocalImage reserves at most one early refresh when a new local
// image disagrees with registry evidence collected for an older image. A cache
// hit alone must never assert that an unseen replacement has a pending update.
// The reservation and cooldown are shared by all containers on this tag/platform.
// It also returns the cache generation, read under the same lock as any
// reservation, so a reservation lives in that generation's map and ForceCheck
// drops it with that map rather than leaving it stuck in the new one.
func (r *RegistryChecker) cachedForLocalImage(key, current string) (*cacheEntry, bool, uint64) {
	r.cache.mu.Lock()
	defer r.cache.mu.Unlock()

	generation := r.cache.generation
	now := time.Now()
	entry, ok := r.cache.entries[key]
	if !ok || now.After(entry.expiresAt) {
		return nil, false, generation
	}
	// Local churn must not bypass registry error/rate-limit backoff. A local
	// digest matching either registry layer already proves it is current.
	if entry.err != "" {
		return &entry, false, generation
	}
	if !r.digestsDiffer(current, entry.comparisonDigests) ||
		!r.digestsDiffer(current, entry.observedLocalDigests) {
		entry.observedLocalDigests = mergeObservedLocalDigests(current, entry.observedLocalDigests)
		r.cache.entries[key] = entry // Do not slide expiry or the refresh budget.
		return &entry, false, generation
	}
	if entry.refreshing || now.Before(entry.refreshAfter) {
		// Unknown is not an affirmative clear. Existing alert consumers retain
		// wanted incidents on errors, and the UI does not offer a false update.
		return &cacheEntry{err: imageChangeRefreshPending}, false, generation
	}
	entry.refreshing = true
	entry.refreshAfter = now.Add(localImageRefreshInterval)
	r.cache.entries[key] = entry
	return &entry, true, generation
}

// clearedDuringLookup reports a lookup whose cache generation ForceCheck
// cleared while the registry request ran. Its answer or error predates the
// manual check, so it is neither cached nor reported: nil leaves the
// container's update status unknown for this cycle, and the next check, after
// the clear, asks the registry again.
func (r *RegistryChecker) clearedDuringLookup(image string) *ImageUpdateResult {
	r.logger.Debug().Str("image", image).Msg("Registry cache was cleared during this lookup; discarding its result")
	return nil
}

// Keep recent local digest identities, not container IDs. Reordering Docker's
// RepoDigests or reporting the index instead of its platform must not cause
// another lookup. Bounded history also prevents long-running local churn from
// retaining every image ever seen; an evicted identity gets the same cooldown.
func mergeObservedLocalDigests(current, previous string) string {
	seen := make(map[string]struct{})
	var digests []string
	for _, list := range []string{current, previous} {
		for _, value := range strings.Split(list, ",") {
			digest := normalizeComparisonDigest(value)
			if digest == "" {
				continue
			}
			if _, ok := seen[digest]; ok {
				continue
			}
			seen[digest] = struct{}{}
			digests = append(digests, digest)
			if len(digests) == maxObservedLocalDigests {
				return strings.Join(digests, ",")
			}
		}
	}
	return strings.Join(digests, ",")
}

// cacheImageDigestResult stores a lookup's answer unless ForceCheck has cleared
// the cache since generation, and reports whether it did.
func (r *RegistryChecker) cacheImageDigestResult(generation uint64, key, latestDigest, comparisonDigests, current string, previous *cacheEntry) bool {
	entry := cacheEntry{
		latestDigest:         latestDigest,
		comparisonDigests:    comparisonDigests,
		expiresAt:            time.Now().Add(defaultCacheTTL),
		observedLocalDigests: mergeObservedLocalDigests(current, ""),
	}
	if previous != nil {
		entry.observedLocalDigests = mergeObservedLocalDigests(current, previous.observedLocalDigests)
		entry.refreshAfter = time.Now().Add(localImageRefreshInterval)
	}
	r.cache.mu.Lock()
	defer r.cache.mu.Unlock()
	if r.cache.generation != generation {
		return false
	}
	r.cache.entries[key] = entry
	return true
}

func (r *RegistryChecker) cacheDigest(key, digest string) {
	r.cacheDigestResult(key, digest, digest)
}

func (r *RegistryChecker) cacheDigestResult(key, latestDigest, comparisonDigests string) {
	r.cache.mu.Lock()
	defer r.cache.mu.Unlock()

	r.cache.entries[key] = cacheEntry{
		latestDigest:      latestDigest,
		comparisonDigests: comparisonDigests,
		expiresAt:         time.Now().Add(defaultCacheTTL),
	}
}

func (r *RegistryChecker) cacheError(key, errMsg string) {
	r.cache.mu.Lock()
	defer r.cache.mu.Unlock()

	r.cache.entries[key] = cacheErrorEntry(errMsg)
}

// cacheErrorInGeneration stores a lookup's error unless ForceCheck has cleared
// the cache since generation, and reports whether it did.
func (r *RegistryChecker) cacheErrorInGeneration(generation uint64, key, errMsg string) bool {
	r.cache.mu.Lock()
	defer r.cache.mu.Unlock()
	if r.cache.generation != generation {
		return false
	}
	r.cache.entries[key] = cacheErrorEntry(errMsg)
	return true
}

func cacheErrorEntry(errMsg string) cacheEntry {
	ttl := errorCacheTTL
	if strings.Contains(strings.ToLower(errMsg), "rate limit") {
		ttl = rateLimitCacheTTL
	}

	return cacheEntry{
		err:       errMsg,
		expiresAt: time.Now().Add(ttl),
	}
}

// CleanupCache removes expired entries from the cache.
func (r *RegistryChecker) CleanupCache() {
	r.cache.mu.Lock()
	defer r.cache.mu.Unlock()

	now := time.Now()
	for key, entry := range r.cache.entries {
		if now.After(entry.expiresAt) {
			delete(r.cache.entries, key)
		}
	}
}

// parseImageReference parses an image reference into registry, repository, and tag.
func parseImageReference(image string) (registry, repository, tag string) {
	// Default values
	registry = "registry-1.docker.io"
	tag = "latest"

	// Check if this is a digest or digest-pinned image (image@sha256: or sha256:...)
	if strings.Contains(image, "@sha256:") || strings.HasPrefix(image, "sha256:") || isValidDigest(image) {
		return "", "", ""
	}

	// Also check for 64-character hex strings (often used as image IDs)
	if len(image) == 64 {
		isHex := true
		for _, c := range image {
			if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
				isHex = false
				break
			}
		}
		if isHex {
			return "", "", ""
		}
	}

	// Split off the tag first
	parts := strings.Split(image, ":")
	if len(parts) > 1 {
		// Check if the last part looks like a tag (not a port)
		lastPart := parts[len(parts)-1]
		if !strings.Contains(lastPart, "/") {
			tag = lastPart
			image = strings.Join(parts[:len(parts)-1], ":")
		}
	}

	// Now parse the registry and repository
	parts = strings.Split(image, "/")

	// If first part looks like a registry (contains . or :, or is localhost)
	if len(parts) > 1 && (strings.Contains(parts[0], ".") || strings.Contains(parts[0], ":") || parts[0] == "localhost") {
		registry = parts[0]
		repository = strings.Join(parts[1:], "/")
	} else if len(parts) == 1 {
		// Official image (e.g., "nginx")
		repository = "library/" + parts[0]
	} else {
		// Docker Hub with namespace (e.g., "myrepo/myapp")
		repository = image
	}

	if isDockerHubRegistry(registry) {
		registry = "registry-1.docker.io"
		// Explicit official-image references need the same library namespace
		// as their short form; never add it to a custom registry's repository.
		if !strings.Contains(repository, "/") {
			repository = "library/" + repository
		}
	}

	return registry, repository, tag
}

// isValidDigest checks if a string looks like a valid digest.
var digestPattern = regexp.MustCompile(`^sha256:[a-f0-9]{64}$`)

func isValidDigest(s string) bool {
	return digestPattern.MatchString(s)
}