Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Refresh Docker tag evidence after a local image replacement

pulse-triage[bot]cef17efda5ff
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 9a689abfe4..5b75adf6a9 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -15,6 +15,19 @@
## Purpose
+### Image comparison refresh is not container lifecycle authority
+
+Docker/Podman collection can refresh stale registry evidence after a changed
+local RepoDigest without a manual update or container action. A one-minute
+per-reference/platform early-refresh budget and in-flight reservation prevent
+rolling replicas or rapid local churn from repeatedly refetching the same tag.
+Unseen mismatches waiting for that budget report unknown, not a false update
+or affirmative clear. Existing transient/rate-limit backoff, image matching,
+credential scope and action/enrolment boundaries stay intact. The monitoring
+contract and collector/registry replacement tests cover the comparison; no
+container recreate, command, restart or installed-recovery authority is added.
+
+
### API-backed disk rates preserve native-writer authority
The physical-disk sync excludes Agent/Proxmox native writers and mock-owned
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index 0423e60df8..990ce413eb 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -17,6 +17,34 @@
## Purpose
+### Docker image replacements invalidate older tag comparisons
+
+Successful tag/platform results are cached for six hours. If an unseen local
+RepoDigest disagrees with that result, ordinary collection refreshes the registry
+instead of asserting an update against evidence for the previous image (#2048).
+An index or platform match is sufficient and needs no refetch. Old/new replicas
+share the latest comparison and bounded local-digest history (64 digests), not a
+cache per container. Reordering or duplicating RepoDigests does not invalidate it.
+
+The first replacement can refresh immediately; further mismatching unseen local
+images share one refresh per minute per reference/platform, with an in-flight
+reservation. While waiting, the status is unknown (`local image changed; registry
+refresh pending`), with no target digest or false update. It is not an affirmative
+clear: existing update-alert error handling preserves wanted incidents. Registry
+failures keep their existing 15-minute transient / one-hour rate-limit backoff,
+regardless of local churn. Ordinary hits extend neither TTL nor cooldown; expiry
+and explicit ForceCheck retain their normal meanings. Registry authentication,
+redaction, manifest validation and public report shapes are unchanged.
+
+`image_replacement_test.go` carries real collection of both reported reference
+forms through ingestion, wanted-alert evaluation and scoped HTTP resource reads.
+`registry_image_change_test.go` pins rolling replicas, rapid churn, reference/OS/
+architecture/variant isolation, concurrency, error backoff and bounded history.
+The Docker/registry package suites remain controls. Synthetic transport and
+browser fixtures establish the source path, not original installed recovery,
+notification receipt, a release, or resolution of every #2048 symptom.
+
+
### Accepted storage listings survive a later child-budget timeout
PVE storage polling commits listings accepted before a later node, content,
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 2a17e3108e..3e89a8a06d 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6388,6 +6388,7 @@
"internal/dockeragent/collect.go",
"internal/dockeragent/container_filesystems.go",
"internal/dockeragent/docker_client.go",
+ "internal/dockeragent/registry.go",
"internal/dockeragent/swarm.go",
"internal/kubernetesagent/agent.go",
"internal/mock/action_fixtures.go",
@@ -7094,6 +7095,23 @@
"internal/monitoring/docker_filesystem_evidence_test.go",
"pkg/agents/filesystem/report_test.go"
]
+ },
+ {
+ "id": "docker-registry-comparison-runtime",
+ "label": "post-replacement registry comparison, bounded refresh and consuming report proof",
+ "match_prefixes": [],
+ "match_files": [
+ "internal/dockeragent/registry.go"
+ ],
+ "allow_same_subsystem_tests": false,
+ "test_prefixes": [],
+ "exact_files": [
+ "internal/dockeragent/image_replacement_test.go",
+ "internal/dockeragent/registry_credentials_test.go",
+ "internal/dockeragent/registry_image_change_test.go",
+ "internal/dockeragent/registry_manifest_test.go",
+ "internal/dockeragent/registry_response_validation_test.go"
+ ]
}
],
"match_files": null
diff --git a/internal/dockeragent/image_replacement_test.go b/internal/dockeragent/image_replacement_test.go
new file mode 100644
index 0000000000..f8ef7fc834
--- /dev/null
+++ b/internal/dockeragent/image_replacement_test.go
@@ -0,0 +1,211 @@
+package dockeragent
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ containertypes "github.com/moby/moby/api/types/container"
+ "github.com/moby/moby/api/types/image"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/api"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/mock"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+ agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
+ "github.com/rs/zerolog"
+)
+
+// Regression for #2048. The Docker reads and registry responses are synthetic;
+// collection, report ingestion, alert evaluation and HTTP projection are real.
+// Alert delay is not bypassed; a separate alert-consumer proof ages these exact
+// produced hosts to exercise pending-incident opening, retention and clearing.
+// An optional JSON log lets browser proof consume the produced HTTP response.
+func TestCollectContainerImageReplacementRefreshesRegistry(t *testing.T) {
+ t.Setenv("PULSE_DEV", "true")
+ previousMock := mock.IsMockEnabled()
+ if err := mock.SetEnabled(false); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := mock.SetEnabled(previousMock); err != nil {
+ t.Error(err)
+ }
+ })
+ for _, ref := range []string{"binwiederhier/ntfy:latest", "ghcr.io/mollyim/mollysocket:1"} {
+ t.Run(ref, func(t *testing.T) {
+ const readerToken = "image-replacement-read-fixture.12345678"
+ reader, err := config.NewAPITokenRecord(readerToken, "Image replacement fixture", []string{config.ScopeMonitoringRead})
+ if err != nil {
+ t.Fatal(err)
+ }
+ dir := t.TempDir()
+ cfg := &config.Config{DataPath: dir, ConfigPath: dir, APITokens: []config.APITokenRecord{*reader}}
+ monitor, err := monitoring.New(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(monitor.Stop)
+ router := api.NewRouter(cfg, monitor, nil, nil, nil, "test")
+ t.Cleanup(router.ShutdownResourceStores)
+ t.Cleanup(router.ShutdownRBAC)
+ t.Cleanup(router.ShutdownBackgroundWorkers)
+ alertCfg := monitor.GetAlertManager().GetConfig()
+ alertCfg.Enabled = true
+ alertCfg.ActivationState = alerts.ActivationActive
+ alertCfg.Schedule.QuietHours.Enabled = false
+ alertCfg.DockerDefaults.UpdateAlertDelayHours = 0
+ monitor.GetAlertManager().UpdateConfig(alertCfg)
+
+ old := "sha256:" + strings.Repeat("1", 64)
+ updated := "sha256:" + strings.Repeat("2", 64)
+ current, remote, imageID := old, old, "image-before"
+ registry, repo, tag := parseImageReference(ref)
+ key := fmt.Sprintf("%s/%s:%s|amd64/linux/", registry, repo, tag)
+ reads := 0
+ checker := NewRegistryChecker(zerolog.Nop())
+ checker.httpClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ if req.URL.Host == "auth.docker.io" || strings.HasSuffix(req.URL.Path, "/token") {
+ return newStringResponse(200, nil, `{"token":"synthetic-only"}`), nil
+ }
+ if req.URL.Host != registry || req.URL.Path != "/v2/"+repo+"/manifests/"+tag || req.Method != http.MethodHead {
+ return nil, fmt.Errorf("unexpected registry request %s %s", req.Method, req.URL)
+ }
+ reads++
+ return newStringResponse(200, map[string]string{"Content-Type": "application/vnd.oci.image.manifest.v1+json", "Docker-Content-Digest": remote}, ""), nil
+ })}
+ a := &Agent{logger: zerolog.Nop(), runtime: RuntimeDocker, registryChecker: checker, prevContainerCPU: make(map[string]cpuSample), docker: &fakeDockerClient{
+ containerInspectWithRawFn: func(_ context.Context, id string, _ bool) (containertypes.InspectResponse, []byte, error) {
+ inspect := baseInspect()
+ inspect.ID = id
+ inspect.Config.Image = ref
+ return inspect, nil, nil
+ },
+ containerStatsOneShotFn: func(context.Context, string) (dockerStatsResponseReader, error) {
+ return statsReader(t, containertypes.StatsResponse{}), nil
+ },
+ imageInspectWithRawFn: func(_ context.Context, id string) (image.InspectResponse, []byte, error) {
+ if id != imageID {
+ return image.InspectResponse{}, nil, fmt.Errorf("unexpected image %s", id)
+ }
+ return image.InspectResponse{RepoDigests: []string{registry + "/" + repo + "@" + current}, Architecture: "amd64", Os: "linux"}, nil, nil
+ },
+ }}
+ // Keep a wanted pending update on another container throughout.
+ control := agentsdocker.Container{ID: strings.Repeat("c", 64), Name: "wanted-update", Image: "example.test/control:1", State: "running", UpdateStatus: &agentsdocker.UpdateStatus{UpdateAvailable: true, CurrentDigest: old, LatestDigest: updated, LastChecked: time.Now()}}
+ collect := func(phase string, wantUpdate bool, wantReads int, errors ...string) {
+ t.Helper()
+ id := strings.Repeat("a", 64)
+ if imageID != "image-before" {
+ id = strings.Repeat("b", 64)
+ }
+ c, err := a.collectContainer(context.Background(), containertypes.Summary{ID: id, Names: []string{"/updated-service"}, Image: ref, ImageID: imageID, State: "running", Status: "Up"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ s := c.UpdateStatus
+ wantError, wantLatest := "", remote
+ if len(errors) > 0 {
+ wantError, wantLatest = errors[0], ""
+ }
+ if s == nil || s.Error != wantError || s.UpdateAvailable != wantUpdate || s.CurrentDigest != current || s.LatestDigest != wantLatest || reads != wantReads {
+ t.Fatalf("%s: current image compared with stale registry evidence: status=%+v manifest_reads=%d want=%d remote=%s", phase, s, reads, wantReads, remote)
+ }
+ report := agentsdocker.Report{Timestamp: time.Now().UTC(), Agent: agentsdocker.AgentInfo{ID: "image-replacement-host", IntervalSeconds: 30}, Host: agentsdocker.HostInfo{Hostname: "image-replacement-host"}, Containers: []agentsdocker.Container{c, control}}
+ host, err := monitor.ApplyDockerReport(report, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(host.Containers) != 2 || host.Containers[0].UpdateStatus == nil || host.Containers[0].UpdateStatus.LatestDigest != wantLatest || host.Containers[0].UpdateStatus.UpdateAvailable != wantUpdate {
+ t.Fatalf("ingestion changed update evidence: %+v", host)
+ }
+ active := monitor.GetAlertManager().GetActiveAlerts()
+ // Zero is normalised to the existing 24h default, not an instant
+ // update alert. Do not weaken production policy for a fixture.
+ if len(active) != 0 {
+ t.Fatalf("%s: update alerted before its delay: %+v", phase, active)
+ }
+ req := httptest.NewRequest(http.MethodGet, "/api/resources?type=app-container&limit=100", nil)
+ req.Header.Set("X-API-Token", readerToken)
+ rec := httptest.NewRecorder()
+ router.Handler().ServeHTTP(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("resource HTTP %d: %s", rec.Code, rec.Body.String())
+ }
+ var body struct {
+ Data []struct {
+ Name string `json:"name"`
+ Docker struct {
+ UpdateStatus *agentsdocker.UpdateStatus `json:"updateStatus"`
+ } `json:"docker"`
+ } `json:"data"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
+ t.Fatal(err)
+ }
+ var found bool
+ for _, resource := range body.Data {
+ if resource.Name != "updated-service" {
+ continue
+ }
+ got := resource.Docker.UpdateStatus
+ if got == nil || got.UpdateAvailable != wantUpdate || got.CurrentDigest != current || got.LatestDigest != wantLatest || got.Error != wantError {
+ t.Fatalf("HTTP projection changed status: %+v", got)
+ }
+ found = true
+ }
+ if !found {
+ t.Fatalf("resource missing from HTTP: %s", rec.Body.String())
+ }
+ if os.Getenv("PULSE_IMAGE_REPLACEMENT_PROOF") != "" {
+ proof, err := json.Marshal(map[string]any{"ref": ref, "phase": phase, "manifestReads": reads, "resources": json.RawMessage(rec.Body.Bytes()), "alerts": active, "host": host})
+ if err != nil {
+ t.Fatal(err)
+ }
+ fmt.Printf("IMAGE_REPLACEMENT_PROOF %s\n", proof)
+ }
+ }
+ collect("before", false, 1)
+ originalExpiry := checker.getCached(key).expiresAt
+ current, remote, imageID = updated, updated, "image-after"
+ collect("replaced", false, 2)
+ expiry := checker.getCached(key).expiresAt
+ if expiry.Before(originalExpiry) || time.Until(expiry) > defaultCacheTTL {
+ t.Fatal("invalid refresh deadline")
+ }
+ for i := 0; i < 3; i++ {
+ collect("stable", false, 2)
+ }
+ if !checker.getCached(key).expiresAt.Equal(expiry) {
+ t.Fatal("ordinary cache hits extended success TTL")
+ }
+ // A new registry tag still makes a real update after normal expiry.
+ remote = "sha256:" + strings.Repeat("3", 64)
+ checker.cache.mu.Lock()
+ entry := checker.cache.entries[key]
+ entry.expiresAt = time.Now().Add(-time.Second)
+ checker.cache.entries[key] = entry
+ checker.cache.mu.Unlock()
+ collect("new-update", true, 3)
+ checker.ForceCheck()
+ collect("forced-check", true, 4)
+ current, remote, imageID = "sha256:"+strings.Repeat("4", 64), "sha256:"+strings.Repeat("5", 64), "image-after-2"
+ collect("still-outdated", true, 5)
+ current, remote, imageID = "sha256:"+strings.Repeat("6", 64), "sha256:"+strings.Repeat("6", 64), "image-after-3"
+ collect("refresh-pending", false, 5, "local image changed; registry refresh pending")
+ checker.cache.mu.Lock()
+ entry = checker.cache.entries[key]
+ entry.refreshAfter = time.Now().Add(-time.Second)
+ checker.cache.entries[key] = entry
+ checker.cache.mu.Unlock()
+ collect("refresh-complete", false, 6)
+
+ })
+ }
+}
diff --git a/internal/dockeragent/registry.go b/internal/dockeragent/registry.go
index 9d652d1a88..9f2386dab4 100644
--- a/internal/dockeragent/registry.go
+++ b/internal/dockeragent/registry.go
@@ -101,11 +101,20 @@ type cacheEntry struct {
comparisonDigests string
expiresAt time.Time
err string // cached error message
+ // Successes remember local images already compared with this tag. Mixed
+ // old/new replicas must share the refreshed result, not invalidate it in turn.
+ observedLocalDigests string
+ refreshAfter time.Time
+ refreshing bool
}
const (
// DefaultCacheTTL is the default time-to-live for cached digests.
defaultCacheTTL = 6 * time.Hour
+ // A newly observed local image can supersede a success before its TTL.
+ // Bound rapid replacements per reference/platform, not per container.
+ localImageRefreshInterval = time.Minute
+ maxObservedLocalDigests = 64
// ErrorCacheTTL is the TTL for caching errors (shorter to allow retry).
errorCacheTTL = 15 * time.Minute
// RateLimitCacheTTL is the TTL for rate-limited lookups. Retrying these on
@@ -118,6 +127,8 @@ const (
defaultCheckInterval = 6 * time.Hour
)
+const imageChangeRefreshPending = "local image changed; registry refresh pending"
+
// ImageUpdateResult contains the result of an image update check.
type ImageUpdateResult struct {
Image string `json:"image"`
@@ -280,7 +291,8 @@ func (r *RegistryChecker) CheckImageUpdate(ctx context.Context, image, currentDi
cacheKey := fmt.Sprintf("%s/%s:%s|%s/%s/%s", registry, repository, tag, arch, goos, variant)
r.logger.Debug().Str("image", image).Str("cacheKey", cacheKey).Msg("Checking update (internal)")
- if cached := r.getCached(cacheKey); cached != nil {
+ cached, refresh := r.cachedForLocalImage(cacheKey, currentDigest)
+ if cached != nil && !refresh {
r.logger.Debug().Str("image", image).Msg("cache hit for update check")
if cached.err != "" {
return &ImageUpdateResult{
@@ -334,7 +346,7 @@ func (r *RegistryChecker) CheckImageUpdate(ctx context.Context, image, currentDi
}
// Cache the successful result
- r.cacheDigestResult(cacheKey, publicLatestDigest, comparisonDigests)
+ r.cacheImageDigestResult(cacheKey, publicLatestDigest, comparisonDigests, currentDigest, cached)
updateAvailable := r.digestsDiffer(currentDigest, comparisonDigests)
@@ -875,6 +887,83 @@ func (r *RegistryChecker) getCached(key string) *cacheEntry {
return &entry
}
+// cachedForLocalImage reserves at most one early refresh when a new local
+// image disagrees with registry evidence collected for an older image. A cache
+// hit alone must never assert that an unseen replacement has a pending update.
+// The reservation and cooldown are shared by all containers on this tag/platform.
+func (r *RegistryChecker) cachedForLocalImage(key, current string) (*cacheEntry, bool) {
+ r.cache.mu.Lock()
+ defer r.cache.mu.Unlock()
+
+ now := time.Now()
+ entry, ok := r.cache.entries[key]
+ if !ok || now.After(entry.expiresAt) {
+ return nil, false
+ }
+ // Local churn must not bypass registry error/rate-limit backoff. A local
+ // digest matching either registry layer already proves it is current.
+ if entry.err != "" {
+ return &entry, false
+ }
+ if !r.digestsDiffer(current, entry.comparisonDigests) ||
+ !r.digestsDiffer(current, entry.observedLocalDigests) {
+ entry.observedLocalDigests = mergeObservedLocalDigests(current, entry.observedLocalDigests)
+ r.cache.entries[key] = entry // Do not slide expiry or the refresh budget.
+ return &entry, false
+ }
+ if entry.refreshing || now.Before(entry.refreshAfter) {
+ // Unknown is not an affirmative clear. Existing alert consumers retain
+ // wanted incidents on errors, and the UI does not offer a false update.
+ return &cacheEntry{err: imageChangeRefreshPending}, false
+ }
+ entry.refreshing = true
+ entry.refreshAfter = now.Add(localImageRefreshInterval)
+ r.cache.entries[key] = entry
+ return &entry, true
+}
+
+// Keep recent local digest identities, not container IDs. Reordering Docker's
+// RepoDigests or reporting the index instead of its platform must not cause
+// another lookup. Bounded history also prevents long-running local churn from
+// retaining every image ever seen; an evicted identity gets the same cooldown.
+func mergeObservedLocalDigests(current, previous string) string {
+ seen := make(map[string]struct{})
+ var digests []string
+ for _, list := range []string{current, previous} {
+ for _, value := range strings.Split(list, ",") {
+ digest := normalizeComparisonDigest(value)
+ if digest == "" {
+ continue
+ }
+ if _, ok := seen[digest]; ok {
+ continue
+ }
+ seen[digest] = struct{}{}
+ digests = append(digests, digest)
+ if len(digests) == maxObservedLocalDigests {
+ return strings.Join(digests, ",")
+ }
+ }
+ }
+ return strings.Join(digests, ",")
+}
+
+func (r *RegistryChecker) cacheImageDigestResult(key, latestDigest, comparisonDigests, current string, previous *cacheEntry) {
+ entry := cacheEntry{
+ latestDigest: latestDigest,
+ comparisonDigests: comparisonDigests,
+ expiresAt: time.Now().Add(defaultCacheTTL),
+ observedLocalDigests: mergeObservedLocalDigests(current, ""),
+ }
+ if previous != nil {
+ entry.observedLocalDigests = mergeObservedLocalDigests(current, previous.observedLocalDigests)
+ entry.refreshAfter = time.Now().Add(localImageRefreshInterval)
+ }
+ r.cache.mu.Lock()
+ defer r.cache.mu.Unlock()
+ r.cache.entries[key] = entry
+}
+
func (r *RegistryChecker) cacheDigest(key, digest string) {
r.cacheDigestResult(key, digest, digest)
}
diff --git a/internal/dockeragent/registry_image_change_test.go b/internal/dockeragent/registry_image_change_test.go
new file mode 100644
index 0000000000..7b57200fb1
--- /dev/null
+++ b/internal/dockeragent/registry_image_change_test.go
@@ -0,0 +1,180 @@
+package dockeragent
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/rs/zerolog"
+)
+
+func TestRegistryImageChangeRefreshIsBounded(t *testing.T) {
+ checker := NewRegistryChecker(zerolog.Nop())
+ const ref = "example.test/service:1"
+ const key = ref + "|amd64/linux/"
+ remote := "sha256:first"
+ calls := 0
+ checker.httpClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ calls++
+ return newStringResponse(http.StatusOK, map[string]string{"Docker-Content-Digest": remote}, ""), nil
+ })}
+ check := func(local string) *ImageUpdateResult {
+ return checker.CheckImageUpdate(context.Background(), ref, local, "amd64", "linux", "")
+ }
+ if got := check(remote); got.Error != "" || got.UpdateAvailable || calls != 1 {
+ t.Fatalf("initial: %+v calls=%d", got, calls)
+ }
+ remote = "sha256:second"
+ if got := check(remote); got.Error != "" || got.UpdateAvailable || calls != 2 {
+ t.Fatalf("replacement: %+v calls=%d", got, calls)
+ }
+ // Old/new replicas, reordered/duplicate digests and repeated inspections
+ // share the same latest result instead of invalidating each other.
+ expiry := checker.getCached(key).expiresAt
+ for _, local := range []string{"sha256:first", "sha256:second", " second , sha256:first,sha256:second"} {
+ got := check(local)
+ if got.Error != "" || got.LatestDigest != remote || calls != 2 {
+ t.Fatalf("replica %s: %+v calls=%d", local, got, calls)
+ }
+ }
+ if !checker.getCached(key).expiresAt.Equal(expiry) {
+ t.Fatal("cache hits slide expiry")
+ }
+ remote = "sha256:fourth"
+ for _, local := range []string{"sha256:third", "sha256:fifth", "sha256:third"} {
+ got := check(local)
+ if got.Error != imageChangeRefreshPending || got.UpdateAvailable || got.LatestDigest != "" || calls != 2 {
+ t.Fatalf("unseen rapid replacement: %+v calls=%d", got, calls)
+ }
+ }
+ // Move only the synthetic refresh deadline, not wall time or the TTL.
+ checker.cache.mu.Lock()
+ entry := checker.cache.entries[key]
+ entry.refreshAfter = time.Now().Add(-time.Second)
+ checker.cache.entries[key] = entry
+ checker.cache.mu.Unlock()
+ got := check("sha256:third")
+ if got.Error != "" || !got.UpdateAvailable || got.LatestDigest != remote || calls != 3 {
+ t.Fatalf("bounded refresh: %+v calls=%d", got, calls)
+ }
+ for _, local := range []string{"sha256:first", "sha256:second", "sha256:third", remote} {
+ got := check(local)
+ if got.Error != "" || got.UpdateAvailable != (local != remote) || calls != 3 {
+ t.Fatalf("shared refreshed result: %+v calls=%d", got, calls)
+ }
+ }
+ // A reference, architecture, OS or variant is still a separate cache key.
+ for _, args := range [][5]string{{"example.test/other:1", "amd64", "linux", "", "sha256:other"}, {ref, "arm", "linux", "v7", "sha256:arm7"}, {ref, "arm", "linux", "v6", "sha256:arm6"}, {ref, "amd64", "windows", "", "sha256:windows"}} {
+ before := calls
+ got := checker.CheckImageUpdate(context.Background(), args[0], args[4], args[1], args[2], args[3])
+ if got.Error != "" || !got.UpdateAvailable || calls != before+1 {
+ t.Fatalf("isolated key: %+v calls=%d", got, calls)
+ }
+ }
+}
+
+func TestRegistryImageChangePreservesErrorBackoff(t *testing.T) {
+ for _, status := range []int{http.StatusTooManyRequests, http.StatusBadGateway} {
+ t.Run(fmt.Sprint(status), func(t *testing.T) {
+ checker := NewRegistryChecker(zerolog.Nop())
+ const ref = "example.test/service:1"
+ const key = ref + "|amd64/linux/"
+ calls := 0
+ fail := false
+ checker.httpClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ calls++
+ if fail {
+ return newStringResponse(status, nil, ""), nil
+ }
+ return newStringResponse(200, map[string]string{"Docker-Content-Digest": "sha256:current"}, ""), nil
+ })}
+ check := func(local string) *ImageUpdateResult {
+ return checker.CheckImageUpdate(context.Background(), ref, local, "amd64", "linux", "")
+ }
+ check("sha256:current")
+ fail = true
+ got := check("sha256:new-local")
+ if got.Error == "" || got.UpdateAvailable || got.LatestDigest != "" || calls != 2 {
+ t.Fatalf("refresh failure: %+v calls=%d", got, calls)
+ }
+ expiry := checker.getCached(key).expiresAt
+ ttl := errorCacheTTL
+ if status == http.StatusTooManyRequests {
+ ttl = rateLimitCacheTTL
+ }
+ if d := time.Until(expiry); d > ttl || d < ttl-time.Second {
+ t.Fatalf("wrong backoff: %s want %s", d, ttl)
+ }
+ for _, local := range []string{"sha256:current", "sha256:newer-local", "sha256:yet-another"} {
+ next := check(local)
+ if next.Error != got.Error || next.UpdateAvailable || next.LatestDigest != "" || calls != 2 {
+ t.Fatalf("local change bypasses error backoff: %+v calls=%d", next, calls)
+ }
+ }
+ if !checker.getCached(key).expiresAt.Equal(expiry) {
+ t.Fatal("errors slide backoff")
+ }
+ checker.cache.mu.Lock()
+ entry := checker.cache.entries[key]
+ entry.expiresAt = time.Now().Add(-time.Second)
+ checker.cache.entries[key] = entry
+ checker.cache.mu.Unlock()
+ fail = false
+ if next := check("sha256:current"); next.Error != "" || next.UpdateAvailable || calls != 3 {
+ t.Fatalf("expiry recovery: %+v calls=%d", next, calls)
+ }
+ })
+ }
+}
+
+func TestRegistryImageChangeRefreshReservation(t *testing.T) {
+ checker := NewRegistryChecker(zerolog.Nop())
+ var calls atomic.Int32
+ started, release := make(chan struct{}), make(chan struct{})
+ checker.httpClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ if calls.Add(1) == 1 {
+ return newStringResponse(200, map[string]string{"Docker-Content-Digest": "sha256:old"}, ""), nil
+ }
+ close(started)
+ <-release
+ return newStringResponse(200, map[string]string{"Docker-Content-Digest": "sha256:new"}, ""), nil
+ })}
+ const ref = "example.test/service:1"
+ check := func() *ImageUpdateResult {
+ return checker.CheckImageUpdate(context.Background(), ref, "sha256:new", "amd64", "linux", "")
+ }
+ checker.CheckImageUpdate(context.Background(), ref, "sha256:old", "amd64", "linux", "")
+ done := make(chan *ImageUpdateResult, 1)
+ go func() { done <- check() }()
+ <-started
+ concurrent := check()
+ close(release)
+ got := <-done
+ if concurrent.Error != imageChangeRefreshPending || concurrent.UpdateAvailable || concurrent.LatestDigest != "" || calls.Load() != 2 {
+ t.Fatalf("concurrent refresh escaped reservation: %+v calls=%d", concurrent, calls.Load())
+ }
+ if got.Error != "" || got.UpdateAvailable || got.LatestDigest != "sha256:new" {
+ t.Fatalf("reserved result: %+v", got)
+ }
+ if got := check(); got.Error != "" || got.UpdateAvailable || calls.Load() != 2 {
+ t.Fatalf("settled cache: %+v calls=%d", got, calls.Load())
+ }
+}
+
+func TestRegistryImageChangeHistoryBound(t *testing.T) {
+ var history string
+ for i := 0; i < 1000; i++ {
+ history = mergeObservedLocalDigests(fmt.Sprintf("sha256:%d", i), history)
+ }
+ digests := strings.Split(history, ",")
+ if len(digests) != maxObservedLocalDigests || digests[0] != "999" || digests[len(digests)-1] != "936" {
+ t.Fatalf("unbounded/nonrecent history: %q", history)
+ }
+ if got := mergeObservedLocalDigests(" sha256:NEW, new, , sha256:old", "old,new"); got != "new,old" {
+ t.Fatalf("digest normalization/dedup: %q", got)
+ }
+}