Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package alerts
import (
"fmt"
"sync/atomic"
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
// Helper to check if an alert exists with the given ID prefix
func hasAlertWithPrefix(alerts []Alert, prefix string) bool {
for _, a := range alerts {
if len(a.ID) >= len(prefix) && a.ID[:len(prefix)] == prefix {
return true
}
}
return false
}
func TestCheckDockerContainerImageUpdate(t *testing.T) {
// Create a manager for testing
m := NewManager()
// Set the default delay hours - NewManager doesn't set this,
// it's normally set during config loading
m.mu.Lock()
m.config.DockerDefaults.UpdateAlertDelayHours = 24 // Default: alert after 24 hours
m.mu.Unlock()
hostID := "docker-host-1"
containerID := "container-abc123"
resourceID := "docker:" + hostID + "/" + containerID
containerName := "test-container"
instanceName := "docker-instance"
nodeName := "docker-node"
host := models.DockerHost{ID: hostID, DisplayName: "Test Host"}
t.Run("no update status - no alert", func(t *testing.T) {
container := models.DockerContainer{
ID: containerID,
Name: containerName,
Image: "nginx:latest",
UpdateStatus: nil,
}
m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)
alerts := m.GetActiveAlerts()
if hasAlertWithPrefix(alerts, "docker-container-update-"+resourceID) {
t.Error("Expected no alert when UpdateStatus is nil")
}
})
t.Run("update not available - no alert", func(t *testing.T) {
container := models.DockerContainer{
ID: containerID,
Name: containerName,
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: false,
CurrentDigest: "sha256:current",
LatestDigest: "sha256:current",
LastChecked: time.Now(),
},
}
m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)
alerts := m.GetActiveAlerts()
if hasAlertWithPrefix(alerts, "docker-container-update-"+resourceID) {
t.Error("Expected no alert when update is not available")
}
})
t.Run("update detection error - no alert", func(t *testing.T) {
container := models.DockerContainer{
ID: containerID,
Name: containerName,
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: false,
Error: "rate limited",
LastChecked: time.Now(),
},
}
m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)
alerts := m.GetActiveAlerts()
if hasAlertWithPrefix(alerts, "docker-container-update-"+resourceID) {
t.Error("Expected no alert when there's an error in update detection")
}
})
t.Run("update available but not yet past threshold - no alert", func(t *testing.T) {
container := models.DockerContainer{
ID: containerID,
Name: containerName,
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
CurrentDigest: "sha256:olddigest",
LatestDigest: "sha256:newdigest",
LastChecked: time.Now(),
},
}
// First call - should record firstSeen but not alert yet
m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)
alerts := m.GetActiveAlerts()
if hasAlertWithPrefix(alerts, "docker-container-update-"+resourceID) {
t.Error("Expected no alert when update just detected (threshold not reached)")
}
// Verify tracking is set
m.mu.RLock()
_, tracked := m.dockerUpdateFirstSeen[resourceID]
trackingKey := dockerUpdateTrackingKey(host, container)
_, trackedByIdentity := m.dockerUpdateFirstSeenByIdentity[trackingKey]
m.mu.RUnlock()
if !tracked {
t.Error("Expected update to be tracked in dockerUpdateFirstSeen")
}
if !trackedByIdentity {
t.Error("Expected update to be tracked in dockerUpdateFirstSeenByIdentity")
}
})
t.Run("update available and past threshold - creates alert", func(t *testing.T) {
// Use a fresh resource ID for this test to avoid interference
testResourceID := "docker:" + hostID + "/container-threshold-test"
// Set up tracking as if we detected the update 25 hours ago
m.mu.Lock()
m.dockerUpdateFirstSeen[testResourceID] = time.Now().Add(-25 * time.Hour)
m.mu.Unlock()
container := models.DockerContainer{
ID: "container-threshold-test",
Name: "threshold-test-container",
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
CurrentDigest: "sha256:olddigest",
LatestDigest: "sha256:newdigest",
LastChecked: time.Now(),
},
}
m.checkDockerContainerImageUpdate(host, container, testResourceID, "threshold-test-container", instanceName, nodeName)
alerts := m.GetActiveAlerts()
found := false
for _, a := range alerts {
if a.Type == "docker-container-update" {
found = true
if a.Level != AlertLevelWarning {
t.Errorf("Expected level %q, got %q", AlertLevelWarning, a.Level)
}
if got := a.Metadata["canonicalAlertKind"]; got != "severity-threshold" {
t.Errorf("canonicalAlertKind = %v, want severity-threshold", got)
}
if got := a.Metadata["canonicalSpecID"]; got != testResourceID+"-image-update" {
t.Errorf("canonicalSpecID = %v, want %s", got, testResourceID+"-image-update")
}
break
}
}
if !found {
t.Error("Expected alert to be created after threshold")
}
})
t.Run("alert cleared when update applied", func(t *testing.T) {
// Use a fresh resource ID
testResourceID := "docker:" + hostID + "/container-clear-test"
// Set up an existing alert by triggering past threshold
m.mu.Lock()
m.dockerUpdateFirstSeen[testResourceID] = time.Now().Add(-25 * time.Hour)
m.mu.Unlock()
container := models.DockerContainer{
ID: "container-clear-test",
Name: "clear-test-container",
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
CurrentDigest: "sha256:olddigest",
LatestDigest: "sha256:newdigest",
LastChecked: time.Now(),
},
}
// First trigger the alert
m.checkDockerContainerImageUpdate(host, container, testResourceID, "clear-test-container", instanceName, nodeName)
// Now simulate container being updated
container.UpdateStatus = &models.DockerContainerUpdateStatus{
UpdateAvailable: false,
CurrentDigest: "sha256:newdigest",
LatestDigest: "sha256:newdigest",
LastChecked: time.Now(),
}
m.checkDockerContainerImageUpdate(host, container, testResourceID, "clear-test-container", instanceName, nodeName)
// Verify tracking is removed
m.mu.RLock()
_, tracked := m.dockerUpdateFirstSeen[testResourceID]
trackingKey := dockerUpdateTrackingKey(host, container)
_, trackedByIdentity := m.dockerUpdateFirstSeenByIdentity[trackingKey]
m.mu.RUnlock()
if tracked {
t.Error("Expected tracking to be removed when update is applied")
}
if trackedByIdentity {
t.Error("Expected identity tracking to be removed when update is applied")
}
})
t.Run("missing update status preserves existing alert and tracking", func(t *testing.T) {
testResourceID := "docker:" + hostID + "/container-unknown-status"
container := models.DockerContainer{
ID: "container-unknown-status",
Name: "unknown-status-container",
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
CurrentDigest: "sha256:old",
LatestDigest: "sha256:new",
LastChecked: time.Now(),
},
}
trackingKey := dockerUpdateTrackingKey(host, container)
firstSeen := time.Now().Add(-25 * time.Hour)
m.mu.Lock()
m.dockerUpdateFirstSeen[testResourceID] = firstSeen
m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
m.mu.Unlock()
m.checkDockerContainerImageUpdate(host, container, testResourceID, "unknown-status-container", instanceName, nodeName)
canonicalAlertID := buildCanonicalStateID(testResourceID, testResourceID+"-image-update")
m.mu.RLock()
alert := testRequireActiveAlert(t, m, canonicalAlertID)
if alert == nil {
m.mu.RUnlock()
t.Fatalf("expected update alert %q to be active", canonicalAlertID)
}
beforeLastSeen := alert.LastSeen
m.mu.RUnlock()
time.Sleep(10 * time.Millisecond)
container.UpdateStatus = nil
m.checkDockerContainerImageUpdate(host, container, testResourceID, "unknown-status-container", instanceName, nodeName)
m.mu.RLock()
alert = testRequireActiveAlert(t, m, canonicalAlertID)
if alert == nil {
m.mu.RUnlock()
t.Fatalf("expected update alert %q to remain active", canonicalAlertID)
}
_, tracked := m.dockerUpdateFirstSeen[testResourceID]
_, trackedByIdentity := m.dockerUpdateFirstSeenByIdentity[trackingKey]
afterLastSeen := alert.LastSeen
m.mu.RUnlock()
if !tracked || !trackedByIdentity {
t.Fatal("expected update tracking to remain when update status is temporarily unavailable")
}
if !afterLastSeen.After(beforeLastSeen) {
t.Fatalf("expected LastSeen to refresh when update status is missing, before=%s after=%s", beforeLastSeen, afterLastSeen)
}
})
// Agents before #2353 reported "no update" with no current digest when an
// image inspect failed. That is not a comparison, so it must not resolve the
// alert and restart a whole delay window.
t.Run("no update without a current digest preserves existing alert and tracking", func(t *testing.T) {
testResourceID := "docker:" + hostID + "/container-no-local-digest"
container := models.DockerContainer{
ID: "container-no-local-digest",
Name: "no-local-digest-container",
Image: "mongo:7",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
CurrentDigest: "sha256:old",
LatestDigest: "sha256:new",
LastChecked: time.Now(),
},
}
trackingKey := dockerUpdateTrackingKey(host, container)
firstSeen := time.Now().Add(-25 * time.Hour)
m.mu.Lock()
m.dockerUpdateFirstSeen[testResourceID] = firstSeen
m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
m.mu.Unlock()
m.checkDockerContainerImageUpdate(host, container, testResourceID, "no-local-digest-container", instanceName, nodeName)
canonicalAlertID := buildCanonicalStateID(testResourceID, testResourceID+"-image-update")
m.mu.RLock()
alert := testRequireActiveAlert(t, m, canonicalAlertID)
if alert == nil {
m.mu.RUnlock()
t.Fatalf("expected update alert %q to be active", canonicalAlertID)
}
startTime := alert.StartTime
m.mu.RUnlock()
container.UpdateStatus = &models.DockerContainerUpdateStatus{
UpdateAvailable: false,
LatestDigest: "sha256:new",
LastChecked: time.Now(),
}
m.checkDockerContainerImageUpdate(host, container, testResourceID, "no-local-digest-container", instanceName, nodeName)
m.mu.RLock()
alert = testRequireActiveAlert(t, m, canonicalAlertID)
var stillStarted time.Time
if alert != nil {
stillStarted = alert.StartTime
}
trackedAge := m.dockerUpdateFirstSeenByIdentity[trackingKey]
m.mu.RUnlock()
if alert == nil {
t.Fatal("a no-update report without a current digest resolved the alert")
}
if !stillStarted.Equal(startTime) || !trackedAge.Equal(firstSeen) {
t.Fatalf("pending age restarted: alert start %s -> %s, tracking %s", startTime, stillStarted, trackedAge)
}
container.UpdateStatus = &models.DockerContainerUpdateStatus{
UpdateAvailable: false,
CurrentDigest: "sha256:new",
LatestDigest: "sha256:new",
LastChecked: time.Now(),
}
m.checkDockerContainerImageUpdate(host, container, testResourceID, "no-local-digest-container", instanceName, nodeName)
if hasAlertWithPrefix(m.GetActiveAlerts(), canonicalAlertID) {
t.Fatal("a real comparison showing the image current did not resolve the alert")
}
})
t.Run("update detection errors preserve existing alert and tracking", func(t *testing.T) {
testResourceID := "docker:" + hostID + "/container-error-status"
container := models.DockerContainer{
ID: "container-error-status",
Name: "error-status-container",
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
CurrentDigest: "sha256:old",
LatestDigest: "sha256:new",
LastChecked: time.Now(),
},
}
trackingKey := dockerUpdateTrackingKey(host, container)
firstSeen := time.Now().Add(-25 * time.Hour)
m.mu.Lock()
m.dockerUpdateFirstSeen[testResourceID] = firstSeen
m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
m.mu.Unlock()
m.checkDockerContainerImageUpdate(host, container, testResourceID, "error-status-container", instanceName, nodeName)
canonicalAlertID := buildCanonicalStateID(testResourceID, testResourceID+"-image-update")
m.mu.RLock()
alert := testRequireActiveAlert(t, m, canonicalAlertID)
if alert == nil {
m.mu.RUnlock()
t.Fatalf("expected update alert %q to be active", canonicalAlertID)
}
beforeLastSeen := alert.LastSeen
m.mu.RUnlock()
time.Sleep(10 * time.Millisecond)
container.UpdateStatus = &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
Error: "rate limited",
LastChecked: time.Now(),
}
m.checkDockerContainerImageUpdate(host, container, testResourceID, "error-status-container", instanceName, nodeName)
m.mu.RLock()
alert = testRequireActiveAlert(t, m, canonicalAlertID)
if alert == nil {
m.mu.RUnlock()
t.Fatalf("expected update alert %q to remain active after check error", canonicalAlertID)
}
_, tracked := m.dockerUpdateFirstSeen[testResourceID]
_, trackedByIdentity := m.dockerUpdateFirstSeenByIdentity[trackingKey]
afterLastSeen := alert.LastSeen
m.mu.RUnlock()
if !tracked || !trackedByIdentity {
t.Fatal("expected update tracking to remain after update check error")
}
if !afterLastSeen.After(beforeLastSeen) {
t.Fatalf("expected LastSeen to refresh after update check error, before=%s after=%s", beforeLastSeen, afterLastSeen)
}
})
t.Run("disabled by negative delay hours", func(t *testing.T) {
// Create manager with disabled update alerts
m2 := NewManager()
m2.mu.Lock()
m2.config.DockerDefaults.UpdateAlertDelayHours = -1 // Disable
m2.mu.Unlock()
testResourceID := "docker:" + hostID + "/container-disabled-test"
container := models.DockerContainer{
ID: "container-disabled-test",
Name: "disabled-test-container",
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
CurrentDigest: "sha256:olddigest",
LatestDigest: "sha256:newdigest",
LastChecked: time.Now(),
},
}
m2.checkDockerContainerImageUpdate(host, container, testResourceID, "disabled-test-container", instanceName, nodeName)
alerts := m2.GetActiveAlerts()
for _, a := range alerts {
if a.Type == "docker-container-update" {
t.Error("Expected no alert when delay hours is negative (disabled)")
break
}
}
})
}
func TestCheckDockerContainerImageUpdatePreservesDelayAcrossHostIDChange(t *testing.T) {
m := NewManager()
m.mu.Lock()
m.config.DockerDefaults.UpdateAlertDelayHours = 24
m.mu.Unlock()
container := models.DockerContainer{
ID: "container-abc123",
Name: "web",
Image: "nginx:latest",
UpdateStatus: &models.DockerContainerUpdateStatus{
UpdateAvailable: true,
CurrentDigest: "sha256:old",
LatestDigest: "sha256:new",
LastChecked: time.Now(),
},
}
oldHost := models.DockerHost{
ID: "docker-host-old",
AgentID: "agent-stable-1",
DisplayName: "Old Docker Host",
Hostname: "docker.local",
}
newHost := oldHost
newHost.ID = "docker-host-new"
newHost.DisplayName = "New Docker Host"
oldResourceID := DockerContainerResourceID(oldHost.ID, container.ID, "")
m.checkDockerContainerImageUpdate(oldHost, container, oldResourceID, "web", "docker-instance", "docker.local")
firstSeen := time.Now().Add(-25 * time.Hour)
trackingKey := dockerUpdateTrackingKey(oldHost, container)
m.mu.Lock()
m.dockerUpdateFirstSeen[oldResourceID] = firstSeen
m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
m.mu.Unlock()
newResourceID := DockerContainerResourceID(newHost.ID, container.ID, "")
m.checkDockerContainerImageUpdate(newHost, container, newResourceID, "web", "docker-instance", "docker.local")
alertID := "docker-container-update-" + newResourceID
m.mu.RLock()
alert, hasAlert := testLookupActiveAlert(t, m, alertID)
resourceFirstSeen, hasResourceTracking := m.dockerUpdateFirstSeen[newResourceID]
identityFirstSeen, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(newHost, container)]
m.mu.RUnlock()
if !hasAlert {
t.Fatalf("Expected update alert %q after host ID change", alertID)
}
if !hasResourceTracking {
t.Fatalf("Expected resource tracking for new host resource ID")
}
if !hasIdentityTracking {
t.Fatalf("Expected identity tracking to persist across host ID change")
}
const tolerance = 2 * time.Second
if delta := alert.StartTime.Sub(firstSeen); delta < -tolerance || delta > tolerance {
t.Fatalf("Expected alert StartTime near %s, got %s", firstSeen, alert.StartTime)
}
if delta := resourceFirstSeen.Sub(firstSeen); delta < -tolerance || delta > tolerance {
t.Fatalf("Expected resource firstSeen near %s, got %s", firstSeen, resourceFirstSeen)
}
if delta := identityFirstSeen.Sub(firstSeen); delta < -tolerance || delta > tolerance {
t.Fatalf("Expected identity firstSeen near %s, got %s", firstSeen, identityFirstSeen)
}
}
// Note: Update alerts are now a free feature (no license gating).
// The Pro license gating tests were removed when update alerts became free.
func TestDockerUpdateTrackingCleanup(t *testing.T) {
m := NewManager()
hostID := "docker-host-1"
prefix := "docker:" + hostID + "/"
host := models.DockerHost{
ID: hostID,
AgentID: "agent-1",
Hostname: "docker-host-1.local",
}
otherHost := models.DockerHost{
ID: "other-host",
AgentID: "agent-other",
}
c1 := models.DockerContainer{ID: "container-1"}
c2 := models.DockerContainer{ID: "container-2"}
c3 := models.DockerContainer{ID: "container-3"}
otherContainer := models.DockerContainer{ID: "container-x"}
// Add some tracking entries
m.mu.Lock()
m.dockerUpdateFirstSeen[prefix+"container-1"] = time.Now()
m.dockerUpdateFirstSeen[prefix+"container-2"] = time.Now()
m.dockerUpdateFirstSeen[prefix+"container-3"] = time.Now()
m.dockerUpdateFirstSeen["docker:other-host/container-x"] = time.Now()
m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c1)] = time.Now()
m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c2)] = time.Now()
m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c3)] = time.Now()
m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(otherHost, otherContainer)] = time.Now()
m.mu.Unlock()
// Create seen maps with only container-1 and container-2
seen := map[string]struct{}{
prefix + "container-1": {},
prefix + "container-2": {},
// container-3 is removed
}
seenUpdateTracking := map[string]struct{}{
dockerUpdateTrackingKey(host, c1): {},
dockerUpdateTrackingKey(host, c2): {},
}
// Run cleanup
m.cleanupDockerContainerAlertsWithTracking(host, seen, seenUpdateTracking)
// Verify container-3 tracking is removed
m.mu.RLock()
_, hasC1 := m.dockerUpdateFirstSeen[prefix+"container-1"]
_, hasC2 := m.dockerUpdateFirstSeen[prefix+"container-2"]
_, hasC3 := m.dockerUpdateFirstSeen[prefix+"container-3"]
_, hasOther := m.dockerUpdateFirstSeen["docker:other-host/container-x"]
_, hasC1ByIdentity := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c1)]
_, hasC2ByIdentity := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c2)]
_, hasC3ByIdentity := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c3)]
_, hasOtherByIdentity := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(otherHost, otherContainer)]
m.mu.RUnlock()
if !hasC1 {
t.Error("Expected container-1 tracking to remain")
}
if !hasC2 {
t.Error("Expected container-2 tracking to remain")
}
if hasC3 {
t.Error("Expected container-3 tracking to be removed")
}
if !hasOther {
t.Error("Expected other host tracking to remain")
}
if !hasC1ByIdentity {
t.Error("Expected container-1 identity tracking to remain")
}
if !hasC2ByIdentity {
t.Error("Expected container-2 identity tracking to remain")
}
if hasC3ByIdentity {
t.Error("Expected container-3 identity tracking to be removed")
}
if !hasOtherByIdentity {
t.Error("Expected other host identity tracking to remain")
}
}
func seedDockerUpdateAlert(t *testing.T, m *Manager, host models.DockerHost, container models.DockerContainer, resourceID string, firstSeen time.Time) (string, string, string) {
t.Helper()
containerName := dockerContainerDisplayName(container)
alertID := "docker-container-update-" + resourceID
canonicalAlertID := buildCanonicalStateID(resourceID, resourceID+"-image-update")
trackingKey := dockerUpdateTrackingKey(host, container)
alert := &Alert{
ID: alertID,
Type: "docker-container-update",
Level: AlertLevelWarning,
ResourceID: resourceID,
ResourceName: containerName,
Instance: dockerInstanceName(host),
CanonicalState: canonicalAlertID,
Metadata: dockerContainerAlertMetadata(host, container, containerName),
StartTime: firstSeen,
LastSeen: firstSeen,
}
m.mu.Lock()
m.setActiveAlertNoLock(canonicalAlertID, alert)
m.dockerUpdateFirstSeen[resourceID] = firstSeen
m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
m.mu.Unlock()
return alertID, canonicalAlertID, trackingKey
}
func TestUpdateConfigClearsDockerContainerUpdateAlertsWhenDisabled(t *testing.T) {
m := NewManager()
host := models.DockerHost{
ID: "docker-host-1",
DisplayName: "Docker Host",
Hostname: "docker.local",
}
container := models.DockerContainer{
ID: "container-1",
Name: "/frontend",
Image: "nginx:latest",
}
resourceID := DockerContainerResourceID(host.ID, container.ID, "")
firstSeen := time.Now().Add(-48 * time.Hour)
_, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)
config := m.GetConfig()
config.DockerDefaults.UpdateAlertDelayHours = -1
m.UpdateConfig(config)
m.mu.RLock()
_, hasAlert := m.activeAlerts[canonicalAlertID]
_, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
_, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
m.mu.RUnlock()
if hasAlert {
t.Fatalf("expected docker update alert to be cleared when update alerts are disabled")
}
if hasResourceTracking {
t.Fatalf("expected docker update resource tracking to be cleared when update alerts are disabled")
}
if hasIdentityTracking {
t.Fatalf("expected docker update identity tracking to be cleared when update alerts are disabled")
}
}
func TestUpdateConfigKeepsDockerContainerUpdateAlertsWhenStillEnabled(t *testing.T) {
m := NewManager()
host := models.DockerHost{
ID: "docker-host-2",
DisplayName: "Docker Host",
Hostname: "docker-2.local",
}
container := models.DockerContainer{
ID: "container-2",
Name: "/api",
Image: "ghcr.io/example/api:latest",
}
resourceID := DockerContainerResourceID(host.ID, container.ID, "")
firstSeen := time.Now().Add(-30 * time.Hour)
_, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)
config := m.GetConfig()
config.DockerDefaults.UpdateAlertDelayHours = 24
config.DockerDefaults.CPU.Trigger = 95
config.DockerDefaults.CPU.Clear = 90
m.UpdateConfig(config)
m.mu.RLock()
_, hasAlert := m.activeAlerts[canonicalAlertID]
resourceFirstSeen, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
identityFirstSeen, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
m.mu.RUnlock()
if !hasAlert {
t.Fatalf("expected docker update alert to remain active when update alerts are still enabled")
}
if !hasResourceTracking {
t.Fatalf("expected docker update resource tracking to remain when update alerts stay enabled")
}
if !hasIdentityTracking {
t.Fatalf("expected docker update identity tracking to remain when update alerts stay enabled")
}
if !resourceFirstSeen.Equal(firstSeen) {
t.Fatalf("expected docker update resource tracking to preserve firstSeen, got %s want %s", resourceFirstSeen, firstSeen)
}
if !identityFirstSeen.Equal(firstSeen) {
t.Fatalf("expected docker update identity tracking to preserve firstSeen, got %s want %s", identityFirstSeen, firstSeen)
}
}
func TestEvaluateDockerContainerClearsUpdateAlertWhenOverrideDisabled(t *testing.T) {
m := NewManager()
host := models.DockerHost{
ID: "docker-host-3",
DisplayName: "Docker Host",
Hostname: "docker-3.local",
}
container := models.DockerContainer{
ID: "container-3",
Name: "/worker",
Image: "ghcr.io/example/worker:latest",
}
resourceID := DockerContainerResourceID(host.ID, container.ID, "")
firstSeen := time.Now().Add(-36 * time.Hour)
alertID, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)
m.mu.Lock()
m.config.Overrides[resourceID] = ThresholdConfig{Disabled: true}
m.mu.Unlock()
m.evaluateDockerContainer(host, container, resourceID)
m.mu.RLock()
_, hasCanonicalAlert := m.activeAlerts[canonicalAlertID]
_, hasLegacyAlias := m.activeAlertAlias[alertID]
_, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
_, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
m.mu.RUnlock()
if hasCanonicalAlert || hasLegacyAlias {
t.Fatalf("expected docker update alert to be cleared when the container override disables alerts")
}
if hasResourceTracking {
t.Fatalf("expected docker update resource tracking to be cleared when the container override disables alerts")
}
if hasIdentityTracking {
t.Fatalf("expected docker update identity tracking to be cleared when the container override disables alerts")
}
}
func TestDockerUpdateTrackingHostKeyFallbackOrder(t *testing.T) {
tests := []struct {
name string
host models.DockerHost
want string
}{
{
name: "agent id preferred",
host: models.DockerHost{
AgentID: " Agent-01 ",
TokenID: "token-should-not-be-used",
},
want: "agent:agent-01",
},
{
name: "token fallback",
host: models.DockerHost{
TokenID: " Token-01 ",
},
want: "token:token-01",
},
{
name: "machine fallback",
host: models.DockerHost{
MachineID: " Machine-01 ",
},
want: "machine:machine-01",
},
{
name: "hostname fallback",
host: models.DockerHost{
Hostname: " Docker-Host.Local ",
},
want: "hostname:docker-host.local",
},
{
name: "id fallback",
host: models.DockerHost{
ID: " Host-01 ",
},
want: "id:host-01",
},
{
name: "display name fallback",
host: models.DockerHost{
DisplayName: " Primary Host ",
},
want: "name:primary host",
},
{
name: "unknown fallback",
host: models.DockerHost{},
want: "unknown-host",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := dockerUpdateTrackingHostKey(tt.host)
if got != tt.want {
t.Fatalf("expected host key %q, got %q", tt.want, got)
}
})
}
}
func TestDockerUpdateTrackingContainerKeyFallbackOrder(t *testing.T) {
tests := []struct {
name string
container models.DockerContainer
want string
}{
{
name: "id preferred",
container: models.DockerContainer{
ID: " Container-01 ",
Name: "/ignored-name",
Image: "ignored-image",
},
want: "id:container-01",
},
{
name: "name fallback trims leading slash",
container: models.DockerContainer{
Name: " /Web ",
Image: "ignored-image",
},
want: "name:web",
},
{
name: "image fallback",
container: models.DockerContainer{
Name: " ",
Image: " Nginx:1.27 ",
},
want: "image:nginx:1.27",
},
{
name: "unknown fallback",
container: models.DockerContainer{},
want: "unknown-container",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := dockerUpdateTrackingContainerKey(tt.container)
if got != tt.want {
t.Fatalf("expected container key %q, got %q", tt.want, got)
}
})
}
}
// Positive cached reports after restart must refresh the persisted occurrence,
// not silently spend another entire delay window below threshold. Exercise the
// public host checker, both checkpoints, real housekeeping and lifecycle events.
func TestDockerUpdateRestartRestoresActivePendingAge(t *testing.T) {
for _, durable := range []bool{false, true} {
for _, delay := range []int{24, 48} {
for _, acknowledged := range []bool{false, true} {
t.Run(fmt.Sprintf("durable=%t/delay=%d/ack=%t", durable, delay, acknowledged), func(t *testing.T) {
dir := t.TempDir()
m := newCleanupRetentionManager(t, dir, durable)
m.mu.Lock()
m.config.DockerDefaults.UpdateAlertDelayHours = delay
m.mu.Unlock()
pending := &models.DockerContainerUpdateStatus{
UpdateAvailable: true, CurrentDigest: "sha256:old", LatestDigest: "sha256:new",
LastChecked: time.Now().Add(-6 * time.Hour),
}
container := models.DockerContainer{ID: "container", Name: "web", Image: "mongo:7", State: "running", UpdateStatus: pending}
host := models.DockerHost{ID: "host", AgentID: "agent", Hostname: "host", Containers: []models.DockerContainer{container}}
resourceID := DockerContainerResourceID(host.ID, container.ID, "")
canonicalID := buildCanonicalStateID(resourceID, resourceID+"-image-update")
trackingKey := dockerUpdateTrackingKey(host, container)
first := time.Now().Add(-time.Duration(delay+2) * time.Hour)
m.mu.Lock()
m.dockerUpdateFirstSeen[resourceID] = first
m.dockerUpdateFirstSeenByIdentity[trackingKey] = first
m.mu.Unlock()
var initialDeliveries atomic.Int32
m.SetAlertCallback(func(*Alert) { initialDeliveries.Add(1) })
m.CheckDockerHost(host)
if got := m.GetActiveAlerts(); len(got) != 1 || !got[0].StartTime.Equal(first) || initialDeliveries.Load() != 1 {
t.Fatalf("initial pending update = %+v", got)
}
if acknowledged {
if err := m.AcknowledgeAlert(canonicalID, "operator"); err != nil {
t.Fatal(err)
}
}
// Model a saved occurrence awaiting its first report after a long
// restart. Do not sleep for a day or change cleanup thresholds.
m.mu.Lock()
active, _ := m.getActiveAlertNoLock(canonicalID)
aged := active.Clone()
aged.LastSeen = time.Now().Add(-25 * time.Hour)
stored := m.setActiveAlertNoLock(canonicalID, aged)
m.mu.Unlock()
if !stored {
t.Fatal("could not checkpoint the aged occurrence")
}
m.Stop()
restarted := newCleanupRetentionManager(t, dir, durable)
restarted.mu.Lock()
restarted.config.DockerDefaults.UpdateAlertDelayHours = delay
restarted.mu.Unlock()
loaded := restarted.GetActiveAlerts()
if len(loaded) != 1 || !loaded[0].StartTime.Equal(first) || !loaded[0].LastSeen.Equal(aged.LastSeen) {
t.Fatalf("checkpoint did not restore the aged occurrence: %+v", loaded)
}
var firings, resolutions, deliveries atomic.Int32
restarted.SetAlertCallback(func(*Alert) { deliveries.Add(1) })
restarted.SubscribeLifecycleCallback(func(event LifecycleEvent) {
switch event.Type {
case eventlog.TypeFired, eventlog.TypeRefired:
firings.Add(1)
case eventlog.TypeResolved:
resolutions.Add(1)
}
})
observedAfter := time.Now()
for _, status := range []*models.DockerContainerUpdateStatus{pending, nil, {Error: "registry unavailable"}, pending} {
host.Containers[0].UpdateStatus = status
restarted.CheckDockerHost(host)
restarted.cleanupStaleMaps() // Hourly tracking and stale-alert cleanup.
restarted.Cleanup(24 * time.Hour) // Escalation's retention sweep.
got := restarted.GetActiveAlerts()
if len(got) != 1 || got[0].ID != loaded[0].ID || !got[0].StartTime.Equal(first) || got[0].LastSeen.Before(observedAfter) || got[0].Acknowledged != acknowledged || got[0].AckUser != loaded[0].AckUser {
t.Fatalf("restart/report/housekeeping lost continuing occurrence: %+v; firings=%d resolutions=%d", got, firings.Load(), resolutions.Load())
}
if got[0].LastNotified == nil || loaded[0].LastNotified == nil || !got[0].LastNotified.Equal(*loaded[0].LastNotified) {
t.Fatal("continuing report changed notification identity")
}
restarted.mu.RLock()
resourceAge := restarted.dockerUpdateFirstSeen[resourceID]
identityAge := restarted.dockerUpdateFirstSeenByIdentity[trackingKey]
restarted.mu.RUnlock()
if !resourceAge.Equal(first) || !identityAge.Equal(first) {
t.Fatalf("restored pending age = %s/%s, want %s", resourceAge, identityAge, first)
}
}
if firings.Load() != 0 || resolutions.Load() != 0 || deliveries.Load() != 0 {
t.Fatalf("continuation invented transitions/delivery: %d/%d/%d", firings.Load(), resolutions.Load(), deliveries.Load())
}
if got := len(restarted.GetAlertHistory(0)); got != 1 {
t.Fatalf("continuation created %d history occurrences, want one", got)
}
if durable {
events := queryAlertEvents(t, restarted, eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeRefired, eventlog.TypeResolved}})
if len(events) != 1 || events[0].Type != eventlog.TypeFired {
t.Fatalf("restart/cleanup invented durable lifecycle events: %+v", events)
}
}
// A different host with the same container ID cannot inherit age.
otherHost := host
otherHost.ID, otherHost.AgentID, otherHost.Hostname = "other-host", "other-agent", "other-host"
restarted.CheckDockerHost(otherHost)
if got := restarted.GetActiveAlerts(); len(got) != 1 {
t.Fatalf("other host inherited an old occurrence: %+v", got)
}
// Explicit recovery retires age; another pending update must wait
// its normal delay rather than reuse the resolved occurrence.
host.Containers[0].UpdateStatus = &models.DockerContainerUpdateStatus{CurrentDigest: "sha256:new", LatestDigest: "sha256:new", LastChecked: time.Now()}
restarted.CheckDockerHost(host)
if len(restarted.GetActiveAlerts()) != 0 || resolutions.Load() != 1 {
t.Fatal("affirmative recovery did not resolve exactly once")
}
host.Containers[0].UpdateStatus = pending
restarted.CheckDockerHost(host)
if len(restarted.GetActiveAlerts()) != 0 || firings.Load() != 0 || deliveries.Load() != 0 {
t.Fatal("new update skipped its delay or reused a resolved occurrence")
}
restarted.mu.RLock()
newFirst := restarted.dockerUpdateFirstSeenByIdentity[trackingKey]
restarted.mu.RUnlock()
if newFirst.Before(observedAfter) {
t.Fatalf("new update inherited old pending age: %s", newFirst)
}
t.Log("positive/unknown/error reports, both cleanup paths, isolated host and affirmative recovery preserved restart lifecycle")
})
}
}
}
}