Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Refresh Docker tag evidence after a local image replacement
pulse-triage[bot]cef17efda5ff
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex 9a689abfe4..5b75adf6a9 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -15,6 +15,19 @@## Purpose+### Image comparison refresh is not container lifecycle authority++Docker/Podman collection can refresh stale registry evidence after a changed+local RepoDigest without a manual update or container action. A one-minute+per-reference/platform early-refresh budget and in-flight reservation prevent+rolling replicas or rapid local churn from repeatedly refetching the same tag.+Unseen mismatches waiting for that budget report unknown, not a false update+or affirmative clear. Existing transient/rate-limit backoff, image matching,+credential scope and action/enrolment boundaries stay intact. The monitoring+contract and collector/registry replacement tests cover the comparison; no+container recreate, command, restart or installed-recovery authority is added.++### API-backed disk rates preserve native-writer authorityThe physical-disk sync excludes Agent/Proxmox native writers and mock-owneddiff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 0423e60df8..990ce413eb 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -17,6 +17,34 @@## Purpose+### Docker image replacements invalidate older tag comparisons++Successful tag/platform results are cached for six hours. If an unseen local+RepoDigest disagrees with that result, ordinary collection refreshes the registry+instead of asserting an update against evidence for the previous image (#2048).+An index or platform match is sufficient and needs no refetch. Old/new replicas+share the latest comparison and bounded local-digest history (64 digests), not a+cache per container. Reordering or duplicating RepoDigests does not invalidate it.++The first replacement can refresh immediately; further mismatching unseen local+images share one refresh per minute per reference/platform, with an in-flight+reservation. While waiting, the status is unknown (`local image changed; registry+refresh pending`), with no target digest or false update. It is not an affirmative+clear: existing update-alert error handling preserves wanted incidents. Registry+failures keep their existing 15-minute transient / one-hour rate-limit backoff,+regardless of local churn. Ordinary hits extend neither TTL nor cooldown; expiry+and explicit ForceCheck retain their normal meanings. Registry authentication,+redaction, manifest validation and public report shapes are unchanged.++`image_replacement_test.go` carries real collection of both reported reference+forms through ingestion, wanted-alert evaluation and scoped HTTP resource reads.+`registry_image_change_test.go` pins rolling replicas, rapid churn, reference/OS/+architecture/variant isolation, concurrency, error backoff and bounded history.+The Docker/registry package suites remain controls. Synthetic transport and+browser fixtures establish the source path, not original installed recovery,+notification receipt, a release, or resolution of every #2048 symptom.++### Accepted storage listings survive a later child-budget timeoutPVE storage polling commits listings accepted before a later node, content,diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.jsonindex 2a17e3108e..3e89a8a06d 100644--- a/docs/release-control/v6/internal/subsystems/registry.json+++ b/docs/release-control/v6/internal/subsystems/registry.json@@ -6388,6 +6388,7 @@"internal/dockeragent/collect.go","internal/dockeragent/container_filesystems.go","internal/dockeragent/docker_client.go",+ "internal/dockeragent/registry.go","internal/dockeragent/swarm.go","internal/kubernetesagent/agent.go","internal/mock/action_fixtures.go",@@ -7094,6 +7095,23 @@"internal/monitoring/docker_filesystem_evidence_test.go","pkg/agents/filesystem/report_test.go"]+ },+ {+ "id": "docker-registry-comparison-runtime",+ "label": "post-replacement registry comparison, bounded refresh and consuming report proof",+ "match_prefixes": [],+ "match_files": [+ "internal/dockeragent/registry.go"+ ],+ "allow_same_subsystem_tests": false,+ "test_prefixes": [],+ "exact_files": [+ "internal/dockeragent/image_replacement_test.go",+ "internal/dockeragent/registry_credentials_test.go",+ "internal/dockeragent/registry_image_change_test.go",+ "internal/dockeragent/registry_manifest_test.go",+ "internal/dockeragent/registry_response_validation_test.go"+ ]}],"match_files": nulldiff --git a/internal/dockeragent/image_replacement_test.go b/internal/dockeragent/image_replacement_test.gonew file mode 100644index 0000000000..f8ef7fc834--- /dev/null+++ b/internal/dockeragent/image_replacement_test.go@@ -0,0 +1,211 @@+package dockeragent++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "os"+ "strings"+ "testing"+ "time"++ containertypes "github.com/moby/moby/api/types/container"+ "github.com/moby/moby/api/types/image"+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/api"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/mock"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"+ "github.com/rs/zerolog"+)++// Regression for #2048. The Docker reads and registry responses are synthetic;+// collection, report ingestion, alert evaluation and HTTP projection are real.+// Alert delay is not bypassed; a separate alert-consumer proof ages these exact+// produced hosts to exercise pending-incident opening, retention and clearing.+// An optional JSON log lets browser proof consume the produced HTTP response.+func TestCollectContainerImageReplacementRefreshesRegistry(t *testing.T) {+ t.Setenv("PULSE_DEV", "true")+ previousMock := mock.IsMockEnabled()+ if err := mock.SetEnabled(false); err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() {+ if err := mock.SetEnabled(previousMock); err != nil {+ t.Error(err)+ }+ })+ for _, ref := range []string{"binwiederhier/ntfy:latest", "ghcr.io/mollyim/mollysocket:1"} {+ t.Run(ref, func(t *testing.T) {+ const readerToken = "image-replacement-read-fixture.12345678"+ reader, err := config.NewAPITokenRecord(readerToken, "Image replacement fixture", []string{config.ScopeMonitoringRead})+ if err != nil {+ t.Fatal(err)+ }+ dir := t.TempDir()+ cfg := &config.Config{DataPath: dir, ConfigPath: dir, APITokens: []config.APITokenRecord{*reader}}+ monitor, err := monitoring.New(cfg)+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(monitor.Stop)+ router := api.NewRouter(cfg, monitor, nil, nil, nil, "test")+ t.Cleanup(router.ShutdownResourceStores)+ t.Cleanup(router.ShutdownRBAC)+ t.Cleanup(router.ShutdownBackgroundWorkers)+ alertCfg := monitor.GetAlertManager().GetConfig()+ alertCfg.Enabled = true+ alertCfg.ActivationState = alerts.ActivationActive+ alertCfg.Schedule.QuietHours.Enabled = false+ alertCfg.DockerDefaults.UpdateAlertDelayHours = 0+ monitor.GetAlertManager().UpdateConfig(alertCfg)++ old := "sha256:" + strings.Repeat("1", 64)+ updated := "sha256:" + strings.Repeat("2", 64)+ current, remote, imageID := old, old, "image-before"+ registry, repo, tag := parseImageReference(ref)+ key := fmt.Sprintf("%s/%s:%s|amd64/linux/", registry, repo, tag)+ reads := 0+ checker := NewRegistryChecker(zerolog.Nop())+ checker.httpClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {+ if req.URL.Host == "auth.docker.io" || strings.HasSuffix(req.URL.Path, "/token") {+ return newStringResponse(200, nil, `{"token":"synthetic-only"}`), nil+ }+ if req.URL.Host != registry || req.URL.Path != "/v2/"+repo+"/manifests/"+tag || req.Method != http.MethodHead {+ return nil, fmt.Errorf("unexpected registry request %s %s", req.Method, req.URL)+ }+ reads+++ return newStringResponse(200, map[string]string{"Content-Type": "application/vnd.oci.image.manifest.v1+json", "Docker-Content-Digest": remote}, ""), nil+ })}+ a := &Agent{logger: zerolog.Nop(), runtime: RuntimeDocker, registryChecker: checker, prevContainerCPU: make(map[string]cpuSample), docker: &fakeDockerClient{+ containerInspectWithRawFn: func(_ context.Context, id string, _ bool) (containertypes.InspectResponse, []byte, error) {+ inspect := baseInspect()+ inspect.ID = id+ inspect.Config.Image = ref+ return inspect, nil, nil+ },+ containerStatsOneShotFn: func(context.Context, string) (dockerStatsResponseReader, error) {+ return statsReader(t, containertypes.StatsResponse{}), nil+ },+ imageInspectWithRawFn: func(_ context.Context, id string) (image.InspectResponse, []byte, error) {+ if id != imageID {+ return image.InspectResponse{}, nil, fmt.Errorf("unexpected image %s", id)+ }+ return image.InspectResponse{RepoDigests: []string{registry + "/" + repo + "@" + current}, Architecture: "amd64", Os: "linux"}, nil, nil+ },+ }}+ // Keep a wanted pending update on another container throughout.+ control := agentsdocker.Container{ID: strings.Repeat("c", 64), Name: "wanted-update", Image: "example.test/control:1", State: "running", UpdateStatus: &agentsdocker.UpdateStatus{UpdateAvailable: true, CurrentDigest: old, LatestDigest: updated, LastChecked: time.Now()}}+ collect := func(phase string, wantUpdate bool, wantReads int, errors ...string) {+ t.Helper()+ id := strings.Repeat("a", 64)+ if imageID != "image-before" {+ id = strings.Repeat("b", 64)+ }+ c, err := a.collectContainer(context.Background(), containertypes.Summary{ID: id, Names: []string{"/updated-service"}, Image: ref, ImageID: imageID, State: "running", Status: "Up"})+ if err != nil {+ t.Fatal(err)+ }+ s := c.UpdateStatus+ wantError, wantLatest := "", remote+ if len(errors) > 0 {+ wantError, wantLatest = errors[0], ""+ }+ if s == nil || s.Error != wantError || s.UpdateAvailable != wantUpdate || s.CurrentDigest != current || s.LatestDigest != wantLatest || reads != wantReads {+ t.Fatalf("%s: current image compared with stale registry evidence: status=%+v manifest_reads=%d want=%d remote=%s", phase, s, reads, wantReads, remote)+ }+ report := agentsdocker.Report{Timestamp: time.Now().UTC(), Agent: agentsdocker.AgentInfo{ID: "image-replacement-host", IntervalSeconds: 30}, Host: agentsdocker.HostInfo{Hostname: "image-replacement-host"}, Containers: []agentsdocker.Container{c, control}}+ host, err := monitor.ApplyDockerReport(report, nil)+ if err != nil {+ t.Fatal(err)+ }+ if len(host.Containers) != 2 || host.Containers[0].UpdateStatus == nil || host.Containers[0].UpdateStatus.LatestDigest != wantLatest || host.Containers[0].UpdateStatus.UpdateAvailable != wantUpdate {+ t.Fatalf("ingestion changed update evidence: %+v", host)+ }+ active := monitor.GetAlertManager().GetActiveAlerts()+ // Zero is normalised to the existing 24h default, not an instant+ // update alert. Do not weaken production policy for a fixture.+ if len(active) != 0 {+ t.Fatalf("%s: update alerted before its delay: %+v", phase, active)+ }+ req := httptest.NewRequest(http.MethodGet, "/api/resources?type=app-container&limit=100", nil)+ req.Header.Set("X-API-Token", readerToken)+ rec := httptest.NewRecorder()+ router.Handler().ServeHTTP(rec, req)+ if rec.Code != http.StatusOK {+ t.Fatalf("resource HTTP %d: %s", rec.Code, rec.Body.String())+ }+ var body struct {+ Data []struct {+ Name string `json:"name"`+ Docker struct {+ UpdateStatus *agentsdocker.UpdateStatus `json:"updateStatus"`+ } `json:"docker"`+ } `json:"data"`+ }+ if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {+ t.Fatal(err)+ }+ var found bool+ for _, resource := range body.Data {+ if resource.Name != "updated-service" {+ continue+ }+ got := resource.Docker.UpdateStatus+ if got == nil || got.UpdateAvailable != wantUpdate || got.CurrentDigest != current || got.LatestDigest != wantLatest || got.Error != wantError {+ t.Fatalf("HTTP projection changed status: %+v", got)+ }+ found = true+ }+ if !found {+ t.Fatalf("resource missing from HTTP: %s", rec.Body.String())+ }+ if os.Getenv("PULSE_IMAGE_REPLACEMENT_PROOF") != "" {+ proof, err := json.Marshal(map[string]any{"ref": ref, "phase": phase, "manifestReads": reads, "resources": json.RawMessage(rec.Body.Bytes()), "alerts": active, "host": host})+ if err != nil {+ t.Fatal(err)+ }+ fmt.Printf("IMAGE_REPLACEMENT_PROOF %s\n", proof)+ }+ }+ collect("before", false, 1)+ originalExpiry := checker.getCached(key).expiresAt+ current, remote, imageID = updated, updated, "image-after"+ collect("replaced", false, 2)+ expiry := checker.getCached(key).expiresAt+ if expiry.Before(originalExpiry) || time.Until(expiry) > defaultCacheTTL {+ t.Fatal("invalid refresh deadline")+ }+ for i := 0; i < 3; i++ {+ collect("stable", false, 2)+ }+ if !checker.getCached(key).expiresAt.Equal(expiry) {+ t.Fatal("ordinary cache hits extended success TTL")+ }+ // A new registry tag still makes a real update after normal expiry.+ remote = "sha256:" + strings.Repeat("3", 64)+ checker.cache.mu.Lock()+ entry := checker.cache.entries[key]+ entry.expiresAt = time.Now().Add(-time.Second)+ checker.cache.entries[key] = entry+ checker.cache.mu.Unlock()+ collect("new-update", true, 3)+ checker.ForceCheck()+ collect("forced-check", true, 4)+ current, remote, imageID = "sha256:"+strings.Repeat("4", 64), "sha256:"+strings.Repeat("5", 64), "image-after-2"+ collect("still-outdated", true, 5)+ current, remote, imageID = "sha256:"+strings.Repeat("6", 64), "sha256:"+strings.Repeat("6", 64), "image-after-3"+ collect("refresh-pending", false, 5, "local image changed; registry refresh pending")+ checker.cache.mu.Lock()+ entry = checker.cache.entries[key]+ entry.refreshAfter = time.Now().Add(-time.Second)+ checker.cache.entries[key] = entry+ checker.cache.mu.Unlock()+ collect("refresh-complete", false, 6)++ })+ }+}diff --git a/internal/dockeragent/registry.go b/internal/dockeragent/registry.goindex 9d652d1a88..9f2386dab4 100644--- a/internal/dockeragent/registry.go+++ b/internal/dockeragent/registry.go@@ -101,11 +101,20 @@ type cacheEntry struct {comparisonDigests stringexpiresAt time.Timeerr string // cached error message+ // Successes remember local images already compared with this tag. Mixed+ // old/new replicas must share the refreshed result, not invalidate it in turn.+ observedLocalDigests string+ refreshAfter time.Time+ refreshing bool}const (// DefaultCacheTTL is the default time-to-live for cached digests.defaultCacheTTL = 6 * time.Hour+ // A newly observed local image can supersede a success before its TTL.+ // Bound rapid replacements per reference/platform, not per container.+ localImageRefreshInterval = time.Minute+ maxObservedLocalDigests = 64// ErrorCacheTTL is the TTL for caching errors (shorter to allow retry).errorCacheTTL = 15 * time.Minute// RateLimitCacheTTL is the TTL for rate-limited lookups. Retrying these on@@ -118,6 +127,8 @@ const (defaultCheckInterval = 6 * time.Hour)+const imageChangeRefreshPending = "local image changed; registry refresh pending"+// ImageUpdateResult contains the result of an image update check.type ImageUpdateResult struct {Image string `json:"image"`@@ -280,7 +291,8 @@ func (r *RegistryChecker) CheckImageUpdate(ctx context.Context, image, currentDicacheKey := fmt.Sprintf("%s/%s:%s|%s/%s/%s", registry, repository, tag, arch, goos, variant)r.logger.Debug().Str("image", image).Str("cacheKey", cacheKey).Msg("Checking update (internal)")- if cached := r.getCached(cacheKey); cached != nil {+ cached, refresh := r.cachedForLocalImage(cacheKey, currentDigest)+ if cached != nil && !refresh {r.logger.Debug().Str("image", image).Msg("cache hit for update check")if cached.err != "" {return &ImageUpdateResult{@@ -334,7 +346,7 @@ func (r *RegistryChecker) CheckImageUpdate(ctx context.Context, image, currentDi}// Cache the successful result- r.cacheDigestResult(cacheKey, publicLatestDigest, comparisonDigests)+ r.cacheImageDigestResult(cacheKey, publicLatestDigest, comparisonDigests, currentDigest, cached)updateAvailable := r.digestsDiffer(currentDigest, comparisonDigests)@@ -875,6 +887,83 @@ func (r *RegistryChecker) getCached(key string) *cacheEntry {return &entry}+// cachedForLocalImage reserves at most one early refresh when a new local+// image disagrees with registry evidence collected for an older image. A cache+// hit alone must never assert that an unseen replacement has a pending update.+// The reservation and cooldown are shared by all containers on this tag/platform.+func (r *RegistryChecker) cachedForLocalImage(key, current string) (*cacheEntry, bool) {+ r.cache.mu.Lock()+ defer r.cache.mu.Unlock()++ now := time.Now()+ entry, ok := r.cache.entries[key]+ if !ok || now.After(entry.expiresAt) {+ return nil, false+ }+ // Local churn must not bypass registry error/rate-limit backoff. A local+ // digest matching either registry layer already proves it is current.+ if entry.err != "" {+ return &entry, false+ }+ if !r.digestsDiffer(current, entry.comparisonDigests) ||+ !r.digestsDiffer(current, entry.observedLocalDigests) {+ entry.observedLocalDigests = mergeObservedLocalDigests(current, entry.observedLocalDigests)+ r.cache.entries[key] = entry // Do not slide expiry or the refresh budget.+ return &entry, false+ }+ if entry.refreshing || now.Before(entry.refreshAfter) {+ // Unknown is not an affirmative clear. Existing alert consumers retain+ // wanted incidents on errors, and the UI does not offer a false update.+ return &cacheEntry{err: imageChangeRefreshPending}, false+ }+ entry.refreshing = true+ entry.refreshAfter = now.Add(localImageRefreshInterval)+ r.cache.entries[key] = entry+ return &entry, true+}++// Keep recent local digest identities, not container IDs. Reordering Docker's+// RepoDigests or reporting the index instead of its platform must not cause+// another lookup. Bounded history also prevents long-running local churn from+// retaining every image ever seen; an evicted identity gets the same cooldown.+func mergeObservedLocalDigests(current, previous string) string {+ seen := make(map[string]struct{})+ var digests []string+ for _, list := range []string{current, previous} {+ for _, value := range strings.Split(list, ",") {+ digest := normalizeComparisonDigest(value)+ if digest == "" {+ continue+ }+ if _, ok := seen[digest]; ok {+ continue+ }+ seen[digest] = struct{}{}+ digests = append(digests, digest)+ if len(digests) == maxObservedLocalDigests {+ return strings.Join(digests, ",")+ }+ }+ }+ return strings.Join(digests, ",")+}++func (r *RegistryChecker) cacheImageDigestResult(key, latestDigest, comparisonDigests, current string, previous *cacheEntry) {+ entry := cacheEntry{+ latestDigest: latestDigest,+ comparisonDigests: comparisonDigests,+ expiresAt: time.Now().Add(defaultCacheTTL),+ observedLocalDigests: mergeObservedLocalDigests(current, ""),+ }+ if previous != nil {+ entry.observedLocalDigests = mergeObservedLocalDigests(current, previous.observedLocalDigests)+ entry.refreshAfter = time.Now().Add(localImageRefreshInterval)+ }+ r.cache.mu.Lock()+ defer r.cache.mu.Unlock()+ r.cache.entries[key] = entry+}+func (r *RegistryChecker) cacheDigest(key, digest string) {r.cacheDigestResult(key, digest, digest)}diff --git a/internal/dockeragent/registry_image_change_test.go b/internal/dockeragent/registry_image_change_test.gonew file mode 100644index 0000000000..7b57200fb1--- /dev/null+++ b/internal/dockeragent/registry_image_change_test.go@@ -0,0 +1,180 @@+package dockeragent++import (+ "context"+ "fmt"+ "net/http"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rs/zerolog"+)++func TestRegistryImageChangeRefreshIsBounded(t *testing.T) {+ checker := NewRegistryChecker(zerolog.Nop())+ const ref = "example.test/service:1"+ const key = ref + "|amd64/linux/"+ remote := "sha256:first"+ calls := 0+ checker.httpClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {+ calls+++ return newStringResponse(http.StatusOK, map[string]string{"Docker-Content-Digest": remote}, ""), nil+ })}+ check := func(local string) *ImageUpdateResult {+ return checker.CheckImageUpdate(context.Background(), ref, local, "amd64", "linux", "")+ }+ if got := check(remote); got.Error != "" || got.UpdateAvailable || calls != 1 {+ t.Fatalf("initial: %+v calls=%d", got, calls)+ }+ remote = "sha256:second"+ if got := check(remote); got.Error != "" || got.UpdateAvailable || calls != 2 {+ t.Fatalf("replacement: %+v calls=%d", got, calls)+ }+ // Old/new replicas, reordered/duplicate digests and repeated inspections+ // share the same latest result instead of invalidating each other.+ expiry := checker.getCached(key).expiresAt+ for _, local := range []string{"sha256:first", "sha256:second", " second , sha256:first,sha256:second"} {+ got := check(local)+ if got.Error != "" || got.LatestDigest != remote || calls != 2 {+ t.Fatalf("replica %s: %+v calls=%d", local, got, calls)+ }+ }+ if !checker.getCached(key).expiresAt.Equal(expiry) {+ t.Fatal("cache hits slide expiry")+ }+ remote = "sha256:fourth"+ for _, local := range []string{"sha256:third", "sha256:fifth", "sha256:third"} {+ got := check(local)+ if got.Error != imageChangeRefreshPending || got.UpdateAvailable || got.LatestDigest != "" || calls != 2 {+ t.Fatalf("unseen rapid replacement: %+v calls=%d", got, calls)+ }+ }+ // Move only the synthetic refresh deadline, not wall time or the TTL.+ checker.cache.mu.Lock()+ entry := checker.cache.entries[key]+ entry.refreshAfter = time.Now().Add(-time.Second)+ checker.cache.entries[key] = entry+ checker.cache.mu.Unlock()+ got := check("sha256:third")+ if got.Error != "" || !got.UpdateAvailable || got.LatestDigest != remote || calls != 3 {+ t.Fatalf("bounded refresh: %+v calls=%d", got, calls)+ }+ for _, local := range []string{"sha256:first", "sha256:second", "sha256:third", remote} {+ got := check(local)+ if got.Error != "" || got.UpdateAvailable != (local != remote) || calls != 3 {+ t.Fatalf("shared refreshed result: %+v calls=%d", got, calls)+ }+ }+ // A reference, architecture, OS or variant is still a separate cache key.+ for _, args := range [][5]string{{"example.test/other:1", "amd64", "linux", "", "sha256:other"}, {ref, "arm", "linux", "v7", "sha256:arm7"}, {ref, "arm", "linux", "v6", "sha256:arm6"}, {ref, "amd64", "windows", "", "sha256:windows"}} {+ before := calls+ got := checker.CheckImageUpdate(context.Background(), args[0], args[4], args[1], args[2], args[3])+ if got.Error != "" || !got.UpdateAvailable || calls != before+1 {+ t.Fatalf("isolated key: %+v calls=%d", got, calls)+ }+ }+}++func TestRegistryImageChangePreservesErrorBackoff(t *testing.T) {+ for _, status := range []int{http.StatusTooManyRequests, http.StatusBadGateway} {+ t.Run(fmt.Sprint(status), func(t *testing.T) {+ checker := NewRegistryChecker(zerolog.Nop())+ const ref = "example.test/service:1"+ const key = ref + "|amd64/linux/"+ calls := 0+ fail := false+ checker.httpClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {+ calls+++ if fail {+ return newStringResponse(status, nil, ""), nil+ }+ return newStringResponse(200, map[string]string{"Docker-Content-Digest": "sha256:current"}, ""), nil+ })}+ check := func(local string) *ImageUpdateResult {+ return checker.CheckImageUpdate(context.Background(), ref, local, "amd64", "linux", "")+ }+ check("sha256:current")+ fail = true+ got := check("sha256:new-local")+ if got.Error == "" || got.UpdateAvailable || got.LatestDigest != "" || calls != 2 {+ t.Fatalf("refresh failure: %+v calls=%d", got, calls)+ }+ expiry := checker.getCached(key).expiresAt+ ttl := errorCacheTTL+ if status == http.StatusTooManyRequests {+ ttl = rateLimitCacheTTL+ }+ if d := time.Until(expiry); d > ttl || d < ttl-time.Second {+ t.Fatalf("wrong backoff: %s want %s", d, ttl)+ }+ for _, local := range []string{"sha256:current", "sha256:newer-local", "sha256:yet-another"} {+ next := check(local)+ if next.Error != got.Error || next.UpdateAvailable || next.LatestDigest != "" || calls != 2 {+ t.Fatalf("local change bypasses error backoff: %+v calls=%d", next, calls)+ }+ }+ if !checker.getCached(key).expiresAt.Equal(expiry) {+ t.Fatal("errors slide backoff")+ }+ checker.cache.mu.Lock()+ entry := checker.cache.entries[key]+ entry.expiresAt = time.Now().Add(-time.Second)+ checker.cache.entries[key] = entry+ checker.cache.mu.Unlock()+ fail = false+ if next := check("sha256:current"); next.Error != "" || next.UpdateAvailable || calls != 3 {+ t.Fatalf("expiry recovery: %+v calls=%d", next, calls)+ }+ })+ }+}++func TestRegistryImageChangeRefreshReservation(t *testing.T) {+ checker := NewRegistryChecker(zerolog.Nop())+ var calls atomic.Int32+ started, release := make(chan struct{}), make(chan struct{})+ checker.httpClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {+ if calls.Add(1) == 1 {+ return newStringResponse(200, map[string]string{"Docker-Content-Digest": "sha256:old"}, ""), nil+ }+ close(started)+ <-release+ return newStringResponse(200, map[string]string{"Docker-Content-Digest": "sha256:new"}, ""), nil+ })}+ const ref = "example.test/service:1"+ check := func() *ImageUpdateResult {+ return checker.CheckImageUpdate(context.Background(), ref, "sha256:new", "amd64", "linux", "")+ }+ checker.CheckImageUpdate(context.Background(), ref, "sha256:old", "amd64", "linux", "")+ done := make(chan *ImageUpdateResult, 1)+ go func() { done <- check() }()+ <-started+ concurrent := check()+ close(release)+ got := <-done+ if concurrent.Error != imageChangeRefreshPending || concurrent.UpdateAvailable || concurrent.LatestDigest != "" || calls.Load() != 2 {+ t.Fatalf("concurrent refresh escaped reservation: %+v calls=%d", concurrent, calls.Load())+ }+ if got.Error != "" || got.UpdateAvailable || got.LatestDigest != "sha256:new" {+ t.Fatalf("reserved result: %+v", got)+ }+ if got := check(); got.Error != "" || got.UpdateAvailable || calls.Load() != 2 {+ t.Fatalf("settled cache: %+v calls=%d", got, calls.Load())+ }+}++func TestRegistryImageChangeHistoryBound(t *testing.T) {+ var history string+ for i := 0; i < 1000; i++ {+ history = mergeObservedLocalDigests(fmt.Sprintf("sha256:%d", i), history)+ }+ digests := strings.Split(history, ",")+ if len(digests) != maxObservedLocalDigests || digests[0] != "999" || digests[len(digests)-1] != "936" {+ t.Fatalf("unbounded/nonrecent history: %q", history)+ }+ if got := mergeObservedLocalDigests(" sha256:NEW, new, , sha256:old", "old,new"); got != "new,old" {+ t.Fatalf("digest normalization/dedup: %q", got)+ }+}