Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

34.7 KB
Raw Blame
package alerts

import (
	"fmt"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
)

// Helper to check if an alert exists with the given ID prefix
func hasAlertWithPrefix(alerts []Alert, prefix string) bool {
	for _, a := range alerts {
		if len(a.ID) >= len(prefix) && a.ID[:len(prefix)] == prefix {
			return true
		}
	}
	return false
}

func TestCheckDockerContainerImageUpdate(t *testing.T) {
	// Create a manager for testing
	m := NewManager()

	// Set the default delay hours - NewManager doesn't set this,
	// it's normally set during config loading
	m.mu.Lock()
	m.config.DockerDefaults.UpdateAlertDelayHours = 24 // Default: alert after 24 hours
	m.mu.Unlock()

	hostID := "docker-host-1"
	containerID := "container-abc123"
	resourceID := "docker:" + hostID + "/" + containerID
	containerName := "test-container"
	instanceName := "docker-instance"
	nodeName := "docker-node"

	host := models.DockerHost{ID: hostID, DisplayName: "Test Host"}

	t.Run("no update status - no alert", func(t *testing.T) {
		container := models.DockerContainer{
			ID:           containerID,
			Name:         containerName,
			Image:        "nginx:latest",
			UpdateStatus: nil,
		}

		m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)

		alerts := m.GetActiveAlerts()
		if hasAlertWithPrefix(alerts, "docker-container-update-"+resourceID) {
			t.Error("Expected no alert when UpdateStatus is nil")
		}
	})

	t.Run("update not available - no alert", func(t *testing.T) {
		container := models.DockerContainer{
			ID:    containerID,
			Name:  containerName,
			Image: "nginx:latest",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: false,
				CurrentDigest:   "sha256:current",
				LatestDigest:    "sha256:current",
				LastChecked:     time.Now(),
			},
		}

		m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)

		alerts := m.GetActiveAlerts()
		if hasAlertWithPrefix(alerts, "docker-container-update-"+resourceID) {
			t.Error("Expected no alert when update is not available")
		}
	})

	t.Run("update detection error - no alert", func(t *testing.T) {
		container := models.DockerContainer{
			ID:    containerID,
			Name:  containerName,
			Image: "nginx:latest",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: false,
				Error:           "rate limited",
				LastChecked:     time.Now(),
			},
		}

		m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)

		alerts := m.GetActiveAlerts()
		if hasAlertWithPrefix(alerts, "docker-container-update-"+resourceID) {
			t.Error("Expected no alert when there's an error in update detection")
		}
	})

	t.Run("update available but not yet past threshold - no alert", func(t *testing.T) {
		container := models.DockerContainer{
			ID:    containerID,
			Name:  containerName,
			Image: "nginx:latest",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: true,
				CurrentDigest:   "sha256:olddigest",
				LatestDigest:    "sha256:newdigest",
				LastChecked:     time.Now(),
			},
		}

		// First call - should record firstSeen but not alert yet
		m.checkDockerContainerImageUpdate(host, container, resourceID, containerName, instanceName, nodeName)

		alerts := m.GetActiveAlerts()
		if hasAlertWithPrefix(alerts, "docker-container-update-"+resourceID) {
			t.Error("Expected no alert when update just detected (threshold not reached)")
		}

		// Verify tracking is set
		m.mu.RLock()
		_, tracked := m.dockerUpdateFirstSeen[resourceID]
		trackingKey := dockerUpdateTrackingKey(host, container)
		_, trackedByIdentity := m.dockerUpdateFirstSeenByIdentity[trackingKey]
		m.mu.RUnlock()
		if !tracked {
			t.Error("Expected update to be tracked in dockerUpdateFirstSeen")
		}
		if !trackedByIdentity {
			t.Error("Expected update to be tracked in dockerUpdateFirstSeenByIdentity")
		}
	})

	t.Run("update available and past threshold - creates alert", func(t *testing.T) {
		// Use a fresh resource ID for this test to avoid interference
		testResourceID := "docker:" + hostID + "/container-threshold-test"

		// Set up tracking as if we detected the update 25 hours ago
		m.mu.Lock()
		m.dockerUpdateFirstSeen[testResourceID] = time.Now().Add(-25 * time.Hour)
		m.mu.Unlock()

		container := models.DockerContainer{
			ID:    "container-threshold-test",
			Name:  "threshold-test-container",
			Image: "nginx:latest",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: true,
				CurrentDigest:   "sha256:olddigest",
				LatestDigest:    "sha256:newdigest",
				LastChecked:     time.Now(),
			},
		}

		m.checkDockerContainerImageUpdate(host, container, testResourceID, "threshold-test-container", instanceName, nodeName)

		alerts := m.GetActiveAlerts()
		found := false
		for _, a := range alerts {
			if a.Type == "docker-container-update" {
				found = true
				if a.Level != AlertLevelWarning {
					t.Errorf("Expected level %q, got %q", AlertLevelWarning, a.Level)
				}
				if got := a.Metadata["canonicalAlertKind"]; got != "severity-threshold" {
					t.Errorf("canonicalAlertKind = %v, want severity-threshold", got)
				}
				if got := a.Metadata["canonicalSpecID"]; got != testResourceID+"-image-update" {
					t.Errorf("canonicalSpecID = %v, want %s", got, testResourceID+"-image-update")
				}
				break
			}
		}
		if !found {
			t.Error("Expected alert to be created after threshold")
		}
	})

	t.Run("alert cleared when update applied", func(t *testing.T) {
		// Use a fresh resource ID
		testResourceID := "docker:" + hostID + "/container-clear-test"

		// Set up an existing alert by triggering past threshold
		m.mu.Lock()
		m.dockerUpdateFirstSeen[testResourceID] = time.Now().Add(-25 * time.Hour)
		m.mu.Unlock()

		container := models.DockerContainer{
			ID:    "container-clear-test",
			Name:  "clear-test-container",
			Image: "nginx:latest",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: true,
				CurrentDigest:   "sha256:olddigest",
				LatestDigest:    "sha256:newdigest",
				LastChecked:     time.Now(),
			},
		}

		// First trigger the alert
		m.checkDockerContainerImageUpdate(host, container, testResourceID, "clear-test-container", instanceName, nodeName)

		// Now simulate container being updated
		container.UpdateStatus = &models.DockerContainerUpdateStatus{
			UpdateAvailable: false,
			CurrentDigest:   "sha256:newdigest",
			LatestDigest:    "sha256:newdigest",
			LastChecked:     time.Now(),
		}

		m.checkDockerContainerImageUpdate(host, container, testResourceID, "clear-test-container", instanceName, nodeName)

		// Verify tracking is removed
		m.mu.RLock()
		_, tracked := m.dockerUpdateFirstSeen[testResourceID]
		trackingKey := dockerUpdateTrackingKey(host, container)
		_, trackedByIdentity := m.dockerUpdateFirstSeenByIdentity[trackingKey]
		m.mu.RUnlock()
		if tracked {
			t.Error("Expected tracking to be removed when update is applied")
		}
		if trackedByIdentity {
			t.Error("Expected identity tracking to be removed when update is applied")
		}
	})

	t.Run("missing update status preserves existing alert and tracking", func(t *testing.T) {
		testResourceID := "docker:" + hostID + "/container-unknown-status"
		container := models.DockerContainer{
			ID:    "container-unknown-status",
			Name:  "unknown-status-container",
			Image: "nginx:latest",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: true,
				CurrentDigest:   "sha256:old",
				LatestDigest:    "sha256:new",
				LastChecked:     time.Now(),
			},
		}
		trackingKey := dockerUpdateTrackingKey(host, container)
		firstSeen := time.Now().Add(-25 * time.Hour)

		m.mu.Lock()
		m.dockerUpdateFirstSeen[testResourceID] = firstSeen
		m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
		m.mu.Unlock()

		m.checkDockerContainerImageUpdate(host, container, testResourceID, "unknown-status-container", instanceName, nodeName)

		canonicalAlertID := buildCanonicalStateID(testResourceID, testResourceID+"-image-update")
		m.mu.RLock()
		alert := testRequireActiveAlert(t, m, canonicalAlertID)
		if alert == nil {
			m.mu.RUnlock()
			t.Fatalf("expected update alert %q to be active", canonicalAlertID)
		}
		beforeLastSeen := alert.LastSeen
		m.mu.RUnlock()

		time.Sleep(10 * time.Millisecond)
		container.UpdateStatus = nil
		m.checkDockerContainerImageUpdate(host, container, testResourceID, "unknown-status-container", instanceName, nodeName)

		m.mu.RLock()
		alert = testRequireActiveAlert(t, m, canonicalAlertID)
		if alert == nil {
			m.mu.RUnlock()
			t.Fatalf("expected update alert %q to remain active", canonicalAlertID)
		}
		_, tracked := m.dockerUpdateFirstSeen[testResourceID]
		_, trackedByIdentity := m.dockerUpdateFirstSeenByIdentity[trackingKey]
		afterLastSeen := alert.LastSeen
		m.mu.RUnlock()

		if !tracked || !trackedByIdentity {
			t.Fatal("expected update tracking to remain when update status is temporarily unavailable")
		}
		if !afterLastSeen.After(beforeLastSeen) {
			t.Fatalf("expected LastSeen to refresh when update status is missing, before=%s after=%s", beforeLastSeen, afterLastSeen)
		}
	})

	// Agents before #2353 reported "no update" with no current digest when an
	// image inspect failed. That is not a comparison, so it must not resolve the
	// alert and restart a whole delay window.
	t.Run("no update without a current digest preserves existing alert and tracking", func(t *testing.T) {
		testResourceID := "docker:" + hostID + "/container-no-local-digest"
		container := models.DockerContainer{
			ID:    "container-no-local-digest",
			Name:  "no-local-digest-container",
			Image: "mongo:7",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: true,
				CurrentDigest:   "sha256:old",
				LatestDigest:    "sha256:new",
				LastChecked:     time.Now(),
			},
		}
		trackingKey := dockerUpdateTrackingKey(host, container)
		firstSeen := time.Now().Add(-25 * time.Hour)
		m.mu.Lock()
		m.dockerUpdateFirstSeen[testResourceID] = firstSeen
		m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
		m.mu.Unlock()

		m.checkDockerContainerImageUpdate(host, container, testResourceID, "no-local-digest-container", instanceName, nodeName)
		canonicalAlertID := buildCanonicalStateID(testResourceID, testResourceID+"-image-update")
		m.mu.RLock()
		alert := testRequireActiveAlert(t, m, canonicalAlertID)
		if alert == nil {
			m.mu.RUnlock()
			t.Fatalf("expected update alert %q to be active", canonicalAlertID)
		}
		startTime := alert.StartTime
		m.mu.RUnlock()

		container.UpdateStatus = &models.DockerContainerUpdateStatus{
			UpdateAvailable: false,
			LatestDigest:    "sha256:new",
			LastChecked:     time.Now(),
		}
		m.checkDockerContainerImageUpdate(host, container, testResourceID, "no-local-digest-container", instanceName, nodeName)

		m.mu.RLock()
		alert = testRequireActiveAlert(t, m, canonicalAlertID)
		var stillStarted time.Time
		if alert != nil {
			stillStarted = alert.StartTime
		}
		trackedAge := m.dockerUpdateFirstSeenByIdentity[trackingKey]
		m.mu.RUnlock()
		if alert == nil {
			t.Fatal("a no-update report without a current digest resolved the alert")
		}
		if !stillStarted.Equal(startTime) || !trackedAge.Equal(firstSeen) {
			t.Fatalf("pending age restarted: alert start %s -> %s, tracking %s", startTime, stillStarted, trackedAge)
		}

		container.UpdateStatus = &models.DockerContainerUpdateStatus{
			UpdateAvailable: false,
			CurrentDigest:   "sha256:new",
			LatestDigest:    "sha256:new",
			LastChecked:     time.Now(),
		}
		m.checkDockerContainerImageUpdate(host, container, testResourceID, "no-local-digest-container", instanceName, nodeName)
		if hasAlertWithPrefix(m.GetActiveAlerts(), canonicalAlertID) {
			t.Fatal("a real comparison showing the image current did not resolve the alert")
		}
	})

	t.Run("update detection errors preserve existing alert and tracking", func(t *testing.T) {
		testResourceID := "docker:" + hostID + "/container-error-status"
		container := models.DockerContainer{
			ID:    "container-error-status",
			Name:  "error-status-container",
			Image: "nginx:latest",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: true,
				CurrentDigest:   "sha256:old",
				LatestDigest:    "sha256:new",
				LastChecked:     time.Now(),
			},
		}
		trackingKey := dockerUpdateTrackingKey(host, container)
		firstSeen := time.Now().Add(-25 * time.Hour)

		m.mu.Lock()
		m.dockerUpdateFirstSeen[testResourceID] = firstSeen
		m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
		m.mu.Unlock()

		m.checkDockerContainerImageUpdate(host, container, testResourceID, "error-status-container", instanceName, nodeName)

		canonicalAlertID := buildCanonicalStateID(testResourceID, testResourceID+"-image-update")
		m.mu.RLock()
		alert := testRequireActiveAlert(t, m, canonicalAlertID)
		if alert == nil {
			m.mu.RUnlock()
			t.Fatalf("expected update alert %q to be active", canonicalAlertID)
		}
		beforeLastSeen := alert.LastSeen
		m.mu.RUnlock()

		time.Sleep(10 * time.Millisecond)
		container.UpdateStatus = &models.DockerContainerUpdateStatus{
			UpdateAvailable: true,
			Error:           "rate limited",
			LastChecked:     time.Now(),
		}
		m.checkDockerContainerImageUpdate(host, container, testResourceID, "error-status-container", instanceName, nodeName)

		m.mu.RLock()
		alert = testRequireActiveAlert(t, m, canonicalAlertID)
		if alert == nil {
			m.mu.RUnlock()
			t.Fatalf("expected update alert %q to remain active after check error", canonicalAlertID)
		}
		_, tracked := m.dockerUpdateFirstSeen[testResourceID]
		_, trackedByIdentity := m.dockerUpdateFirstSeenByIdentity[trackingKey]
		afterLastSeen := alert.LastSeen
		m.mu.RUnlock()

		if !tracked || !trackedByIdentity {
			t.Fatal("expected update tracking to remain after update check error")
		}
		if !afterLastSeen.After(beforeLastSeen) {
			t.Fatalf("expected LastSeen to refresh after update check error, before=%s after=%s", beforeLastSeen, afterLastSeen)
		}
	})

	t.Run("disabled by negative delay hours", func(t *testing.T) {
		// Create manager with disabled update alerts
		m2 := NewManager()
		m2.mu.Lock()
		m2.config.DockerDefaults.UpdateAlertDelayHours = -1 // Disable
		m2.mu.Unlock()

		testResourceID := "docker:" + hostID + "/container-disabled-test"

		container := models.DockerContainer{
			ID:    "container-disabled-test",
			Name:  "disabled-test-container",
			Image: "nginx:latest",
			UpdateStatus: &models.DockerContainerUpdateStatus{
				UpdateAvailable: true,
				CurrentDigest:   "sha256:olddigest",
				LatestDigest:    "sha256:newdigest",
				LastChecked:     time.Now(),
			},
		}

		m2.checkDockerContainerImageUpdate(host, container, testResourceID, "disabled-test-container", instanceName, nodeName)

		alerts := m2.GetActiveAlerts()
		for _, a := range alerts {
			if a.Type == "docker-container-update" {
				t.Error("Expected no alert when delay hours is negative (disabled)")
				break
			}
		}
	})
}

func TestCheckDockerContainerImageUpdatePreservesDelayAcrossHostIDChange(t *testing.T) {
	m := NewManager()
	m.mu.Lock()
	m.config.DockerDefaults.UpdateAlertDelayHours = 24
	m.mu.Unlock()

	container := models.DockerContainer{
		ID:    "container-abc123",
		Name:  "web",
		Image: "nginx:latest",
		UpdateStatus: &models.DockerContainerUpdateStatus{
			UpdateAvailable: true,
			CurrentDigest:   "sha256:old",
			LatestDigest:    "sha256:new",
			LastChecked:     time.Now(),
		},
	}

	oldHost := models.DockerHost{
		ID:          "docker-host-old",
		AgentID:     "agent-stable-1",
		DisplayName: "Old Docker Host",
		Hostname:    "docker.local",
	}
	newHost := oldHost
	newHost.ID = "docker-host-new"
	newHost.DisplayName = "New Docker Host"

	oldResourceID := DockerContainerResourceID(oldHost.ID, container.ID, "")
	m.checkDockerContainerImageUpdate(oldHost, container, oldResourceID, "web", "docker-instance", "docker.local")

	firstSeen := time.Now().Add(-25 * time.Hour)
	trackingKey := dockerUpdateTrackingKey(oldHost, container)
	m.mu.Lock()
	m.dockerUpdateFirstSeen[oldResourceID] = firstSeen
	m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
	m.mu.Unlock()

	newResourceID := DockerContainerResourceID(newHost.ID, container.ID, "")
	m.checkDockerContainerImageUpdate(newHost, container, newResourceID, "web", "docker-instance", "docker.local")

	alertID := "docker-container-update-" + newResourceID
	m.mu.RLock()
	alert, hasAlert := testLookupActiveAlert(t, m, alertID)
	resourceFirstSeen, hasResourceTracking := m.dockerUpdateFirstSeen[newResourceID]
	identityFirstSeen, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(newHost, container)]
	m.mu.RUnlock()

	if !hasAlert {
		t.Fatalf("Expected update alert %q after host ID change", alertID)
	}
	if !hasResourceTracking {
		t.Fatalf("Expected resource tracking for new host resource ID")
	}
	if !hasIdentityTracking {
		t.Fatalf("Expected identity tracking to persist across host ID change")
	}

	const tolerance = 2 * time.Second
	if delta := alert.StartTime.Sub(firstSeen); delta < -tolerance || delta > tolerance {
		t.Fatalf("Expected alert StartTime near %s, got %s", firstSeen, alert.StartTime)
	}
	if delta := resourceFirstSeen.Sub(firstSeen); delta < -tolerance || delta > tolerance {
		t.Fatalf("Expected resource firstSeen near %s, got %s", firstSeen, resourceFirstSeen)
	}
	if delta := identityFirstSeen.Sub(firstSeen); delta < -tolerance || delta > tolerance {
		t.Fatalf("Expected identity firstSeen near %s, got %s", firstSeen, identityFirstSeen)
	}
}

// Note: Update alerts are now a free feature (no license gating).
// The Pro license gating tests were removed when update alerts became free.

func TestDockerUpdateTrackingCleanup(t *testing.T) {
	m := NewManager()

	hostID := "docker-host-1"
	prefix := "docker:" + hostID + "/"
	host := models.DockerHost{
		ID:       hostID,
		AgentID:  "agent-1",
		Hostname: "docker-host-1.local",
	}
	otherHost := models.DockerHost{
		ID:      "other-host",
		AgentID: "agent-other",
	}
	c1 := models.DockerContainer{ID: "container-1"}
	c2 := models.DockerContainer{ID: "container-2"}
	c3 := models.DockerContainer{ID: "container-3"}
	otherContainer := models.DockerContainer{ID: "container-x"}

	// Add some tracking entries
	m.mu.Lock()
	m.dockerUpdateFirstSeen[prefix+"container-1"] = time.Now()
	m.dockerUpdateFirstSeen[prefix+"container-2"] = time.Now()
	m.dockerUpdateFirstSeen[prefix+"container-3"] = time.Now()
	m.dockerUpdateFirstSeen["docker:other-host/container-x"] = time.Now()
	m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c1)] = time.Now()
	m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c2)] = time.Now()
	m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c3)] = time.Now()
	m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(otherHost, otherContainer)] = time.Now()
	m.mu.Unlock()

	// Create seen maps with only container-1 and container-2
	seen := map[string]struct{}{
		prefix + "container-1": {},
		prefix + "container-2": {},
		// container-3 is removed
	}
	seenUpdateTracking := map[string]struct{}{
		dockerUpdateTrackingKey(host, c1): {},
		dockerUpdateTrackingKey(host, c2): {},
	}

	// Run cleanup
	m.cleanupDockerContainerAlertsWithTracking(host, seen, seenUpdateTracking)

	// Verify container-3 tracking is removed
	m.mu.RLock()
	_, hasC1 := m.dockerUpdateFirstSeen[prefix+"container-1"]
	_, hasC2 := m.dockerUpdateFirstSeen[prefix+"container-2"]
	_, hasC3 := m.dockerUpdateFirstSeen[prefix+"container-3"]
	_, hasOther := m.dockerUpdateFirstSeen["docker:other-host/container-x"]
	_, hasC1ByIdentity := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c1)]
	_, hasC2ByIdentity := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c2)]
	_, hasC3ByIdentity := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(host, c3)]
	_, hasOtherByIdentity := m.dockerUpdateFirstSeenByIdentity[dockerUpdateTrackingKey(otherHost, otherContainer)]
	m.mu.RUnlock()

	if !hasC1 {
		t.Error("Expected container-1 tracking to remain")
	}
	if !hasC2 {
		t.Error("Expected container-2 tracking to remain")
	}
	if hasC3 {
		t.Error("Expected container-3 tracking to be removed")
	}
	if !hasOther {
		t.Error("Expected other host tracking to remain")
	}
	if !hasC1ByIdentity {
		t.Error("Expected container-1 identity tracking to remain")
	}
	if !hasC2ByIdentity {
		t.Error("Expected container-2 identity tracking to remain")
	}
	if hasC3ByIdentity {
		t.Error("Expected container-3 identity tracking to be removed")
	}
	if !hasOtherByIdentity {
		t.Error("Expected other host identity tracking to remain")
	}
}

func seedDockerUpdateAlert(t *testing.T, m *Manager, host models.DockerHost, container models.DockerContainer, resourceID string, firstSeen time.Time) (string, string, string) {
	t.Helper()

	containerName := dockerContainerDisplayName(container)
	alertID := "docker-container-update-" + resourceID
	canonicalAlertID := buildCanonicalStateID(resourceID, resourceID+"-image-update")
	trackingKey := dockerUpdateTrackingKey(host, container)
	alert := &Alert{
		ID:             alertID,
		Type:           "docker-container-update",
		Level:          AlertLevelWarning,
		ResourceID:     resourceID,
		ResourceName:   containerName,
		Instance:       dockerInstanceName(host),
		CanonicalState: canonicalAlertID,
		Metadata:       dockerContainerAlertMetadata(host, container, containerName),
		StartTime:      firstSeen,
		LastSeen:       firstSeen,
	}

	m.mu.Lock()
	m.setActiveAlertNoLock(canonicalAlertID, alert)
	m.dockerUpdateFirstSeen[resourceID] = firstSeen
	m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
	m.mu.Unlock()

	return alertID, canonicalAlertID, trackingKey
}

func TestUpdateConfigClearsDockerContainerUpdateAlertsWhenDisabled(t *testing.T) {
	m := NewManager()

	host := models.DockerHost{
		ID:          "docker-host-1",
		DisplayName: "Docker Host",
		Hostname:    "docker.local",
	}
	container := models.DockerContainer{
		ID:    "container-1",
		Name:  "/frontend",
		Image: "nginx:latest",
	}
	resourceID := DockerContainerResourceID(host.ID, container.ID, "")
	firstSeen := time.Now().Add(-48 * time.Hour)
	_, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)

	config := m.GetConfig()
	config.DockerDefaults.UpdateAlertDelayHours = -1
	m.UpdateConfig(config)

	m.mu.RLock()
	_, hasAlert := m.activeAlerts[canonicalAlertID]
	_, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
	_, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
	m.mu.RUnlock()

	if hasAlert {
		t.Fatalf("expected docker update alert to be cleared when update alerts are disabled")
	}
	if hasResourceTracking {
		t.Fatalf("expected docker update resource tracking to be cleared when update alerts are disabled")
	}
	if hasIdentityTracking {
		t.Fatalf("expected docker update identity tracking to be cleared when update alerts are disabled")
	}
}

func TestUpdateConfigKeepsDockerContainerUpdateAlertsWhenStillEnabled(t *testing.T) {
	m := NewManager()

	host := models.DockerHost{
		ID:          "docker-host-2",
		DisplayName: "Docker Host",
		Hostname:    "docker-2.local",
	}
	container := models.DockerContainer{
		ID:    "container-2",
		Name:  "/api",
		Image: "ghcr.io/example/api:latest",
	}
	resourceID := DockerContainerResourceID(host.ID, container.ID, "")
	firstSeen := time.Now().Add(-30 * time.Hour)
	_, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)

	config := m.GetConfig()
	config.DockerDefaults.UpdateAlertDelayHours = 24
	config.DockerDefaults.CPU.Trigger = 95
	config.DockerDefaults.CPU.Clear = 90
	m.UpdateConfig(config)

	m.mu.RLock()
	_, hasAlert := m.activeAlerts[canonicalAlertID]
	resourceFirstSeen, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
	identityFirstSeen, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
	m.mu.RUnlock()

	if !hasAlert {
		t.Fatalf("expected docker update alert to remain active when update alerts are still enabled")
	}
	if !hasResourceTracking {
		t.Fatalf("expected docker update resource tracking to remain when update alerts stay enabled")
	}
	if !hasIdentityTracking {
		t.Fatalf("expected docker update identity tracking to remain when update alerts stay enabled")
	}
	if !resourceFirstSeen.Equal(firstSeen) {
		t.Fatalf("expected docker update resource tracking to preserve firstSeen, got %s want %s", resourceFirstSeen, firstSeen)
	}
	if !identityFirstSeen.Equal(firstSeen) {
		t.Fatalf("expected docker update identity tracking to preserve firstSeen, got %s want %s", identityFirstSeen, firstSeen)
	}
}

func TestEvaluateDockerContainerClearsUpdateAlertWhenOverrideDisabled(t *testing.T) {
	m := NewManager()

	host := models.DockerHost{
		ID:          "docker-host-3",
		DisplayName: "Docker Host",
		Hostname:    "docker-3.local",
	}
	container := models.DockerContainer{
		ID:    "container-3",
		Name:  "/worker",
		Image: "ghcr.io/example/worker:latest",
	}
	resourceID := DockerContainerResourceID(host.ID, container.ID, "")
	firstSeen := time.Now().Add(-36 * time.Hour)
	alertID, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)

	m.mu.Lock()
	m.config.Overrides[resourceID] = ThresholdConfig{Disabled: true}
	m.mu.Unlock()

	m.evaluateDockerContainer(host, container, resourceID)

	m.mu.RLock()
	_, hasCanonicalAlert := m.activeAlerts[canonicalAlertID]
	_, hasLegacyAlias := m.activeAlertAlias[alertID]
	_, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
	_, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
	m.mu.RUnlock()

	if hasCanonicalAlert || hasLegacyAlias {
		t.Fatalf("expected docker update alert to be cleared when the container override disables alerts")
	}
	if hasResourceTracking {
		t.Fatalf("expected docker update resource tracking to be cleared when the container override disables alerts")
	}
	if hasIdentityTracking {
		t.Fatalf("expected docker update identity tracking to be cleared when the container override disables alerts")
	}
}

func TestDockerUpdateTrackingHostKeyFallbackOrder(t *testing.T) {
	tests := []struct {
		name string
		host models.DockerHost
		want string
	}{
		{
			name: "agent id preferred",
			host: models.DockerHost{
				AgentID: " Agent-01 ",
				TokenID: "token-should-not-be-used",
			},
			want: "agent:agent-01",
		},
		{
			name: "token fallback",
			host: models.DockerHost{
				TokenID: " Token-01 ",
			},
			want: "token:token-01",
		},
		{
			name: "machine fallback",
			host: models.DockerHost{
				MachineID: " Machine-01 ",
			},
			want: "machine:machine-01",
		},
		{
			name: "hostname fallback",
			host: models.DockerHost{
				Hostname: " Docker-Host.Local ",
			},
			want: "hostname:docker-host.local",
		},
		{
			name: "id fallback",
			host: models.DockerHost{
				ID: " Host-01 ",
			},
			want: "id:host-01",
		},
		{
			name: "display name fallback",
			host: models.DockerHost{
				DisplayName: " Primary Host ",
			},
			want: "name:primary host",
		},
		{
			name: "unknown fallback",
			host: models.DockerHost{},
			want: "unknown-host",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := dockerUpdateTrackingHostKey(tt.host)
			if got != tt.want {
				t.Fatalf("expected host key %q, got %q", tt.want, got)
			}
		})
	}
}

func TestDockerUpdateTrackingContainerKeyFallbackOrder(t *testing.T) {
	tests := []struct {
		name      string
		container models.DockerContainer
		want      string
	}{
		{
			name: "id preferred",
			container: models.DockerContainer{
				ID:    " Container-01 ",
				Name:  "/ignored-name",
				Image: "ignored-image",
			},
			want: "id:container-01",
		},
		{
			name: "name fallback trims leading slash",
			container: models.DockerContainer{
				Name:  " /Web ",
				Image: "ignored-image",
			},
			want: "name:web",
		},
		{
			name: "image fallback",
			container: models.DockerContainer{
				Name:  "   ",
				Image: " Nginx:1.27 ",
			},
			want: "image:nginx:1.27",
		},
		{
			name:      "unknown fallback",
			container: models.DockerContainer{},
			want:      "unknown-container",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := dockerUpdateTrackingContainerKey(tt.container)
			if got != tt.want {
				t.Fatalf("expected container key %q, got %q", tt.want, got)
			}
		})
	}
}

// Positive cached reports after restart must refresh the persisted occurrence,
// not silently spend another entire delay window below threshold. Exercise the
// public host checker, both checkpoints, real housekeeping and lifecycle events.
func TestDockerUpdateRestartRestoresActivePendingAge(t *testing.T) {
	for _, durable := range []bool{false, true} {
		for _, delay := range []int{24, 48} {
			for _, acknowledged := range []bool{false, true} {
				t.Run(fmt.Sprintf("durable=%t/delay=%d/ack=%t", durable, delay, acknowledged), func(t *testing.T) {
					dir := t.TempDir()
					m := newCleanupRetentionManager(t, dir, durable)
					m.mu.Lock()
					m.config.DockerDefaults.UpdateAlertDelayHours = delay
					m.mu.Unlock()
					pending := &models.DockerContainerUpdateStatus{
						UpdateAvailable: true, CurrentDigest: "sha256:old", LatestDigest: "sha256:new",
						LastChecked: time.Now().Add(-6 * time.Hour),
					}
					container := models.DockerContainer{ID: "container", Name: "web", Image: "mongo:7", State: "running", UpdateStatus: pending}
					host := models.DockerHost{ID: "host", AgentID: "agent", Hostname: "host", Containers: []models.DockerContainer{container}}
					resourceID := DockerContainerResourceID(host.ID, container.ID, "")
					canonicalID := buildCanonicalStateID(resourceID, resourceID+"-image-update")
					trackingKey := dockerUpdateTrackingKey(host, container)
					first := time.Now().Add(-time.Duration(delay+2) * time.Hour)
					m.mu.Lock()
					m.dockerUpdateFirstSeen[resourceID] = first
					m.dockerUpdateFirstSeenByIdentity[trackingKey] = first
					m.mu.Unlock()
					var initialDeliveries atomic.Int32
					m.SetAlertCallback(func(*Alert) { initialDeliveries.Add(1) })
					m.CheckDockerHost(host)
					if got := m.GetActiveAlerts(); len(got) != 1 || !got[0].StartTime.Equal(first) || initialDeliveries.Load() != 1 {
						t.Fatalf("initial pending update = %+v", got)
					}
					if acknowledged {
						if err := m.AcknowledgeAlert(canonicalID, "operator"); err != nil {
							t.Fatal(err)
						}
					}
					// Model a saved occurrence awaiting its first report after a long
					// restart. Do not sleep for a day or change cleanup thresholds.
					m.mu.Lock()
					active, _ := m.getActiveAlertNoLock(canonicalID)
					aged := active.Clone()
					aged.LastSeen = time.Now().Add(-25 * time.Hour)
					stored := m.setActiveAlertNoLock(canonicalID, aged)
					m.mu.Unlock()
					if !stored {
						t.Fatal("could not checkpoint the aged occurrence")
					}
					m.Stop()

					restarted := newCleanupRetentionManager(t, dir, durable)
					restarted.mu.Lock()
					restarted.config.DockerDefaults.UpdateAlertDelayHours = delay
					restarted.mu.Unlock()
					loaded := restarted.GetActiveAlerts()
					if len(loaded) != 1 || !loaded[0].StartTime.Equal(first) || !loaded[0].LastSeen.Equal(aged.LastSeen) {
						t.Fatalf("checkpoint did not restore the aged occurrence: %+v", loaded)
					}
					var firings, resolutions, deliveries atomic.Int32
					restarted.SetAlertCallback(func(*Alert) { deliveries.Add(1) })
					restarted.SubscribeLifecycleCallback(func(event LifecycleEvent) {
						switch event.Type {
						case eventlog.TypeFired, eventlog.TypeRefired:
							firings.Add(1)
						case eventlog.TypeResolved:
							resolutions.Add(1)
						}
					})
					observedAfter := time.Now()
					for _, status := range []*models.DockerContainerUpdateStatus{pending, nil, {Error: "registry unavailable"}, pending} {
						host.Containers[0].UpdateStatus = status
						restarted.CheckDockerHost(host)
						restarted.cleanupStaleMaps()      // Hourly tracking and stale-alert cleanup.
						restarted.Cleanup(24 * time.Hour) // Escalation's retention sweep.
						got := restarted.GetActiveAlerts()
						if len(got) != 1 || got[0].ID != loaded[0].ID || !got[0].StartTime.Equal(first) || got[0].LastSeen.Before(observedAfter) || got[0].Acknowledged != acknowledged || got[0].AckUser != loaded[0].AckUser {
							t.Fatalf("restart/report/housekeeping lost continuing occurrence: %+v; firings=%d resolutions=%d", got, firings.Load(), resolutions.Load())
						}
						if got[0].LastNotified == nil || loaded[0].LastNotified == nil || !got[0].LastNotified.Equal(*loaded[0].LastNotified) {
							t.Fatal("continuing report changed notification identity")
						}
						restarted.mu.RLock()
						resourceAge := restarted.dockerUpdateFirstSeen[resourceID]
						identityAge := restarted.dockerUpdateFirstSeenByIdentity[trackingKey]
						restarted.mu.RUnlock()
						if !resourceAge.Equal(first) || !identityAge.Equal(first) {
							t.Fatalf("restored pending age = %s/%s, want %s", resourceAge, identityAge, first)
						}
					}
					if firings.Load() != 0 || resolutions.Load() != 0 || deliveries.Load() != 0 {
						t.Fatalf("continuation invented transitions/delivery: %d/%d/%d", firings.Load(), resolutions.Load(), deliveries.Load())
					}
					if got := len(restarted.GetAlertHistory(0)); got != 1 {
						t.Fatalf("continuation created %d history occurrences, want one", got)
					}
					if durable {
						events := queryAlertEvents(t, restarted, eventlog.Filter{Types: []string{eventlog.TypeFired, eventlog.TypeRefired, eventlog.TypeResolved}})
						if len(events) != 1 || events[0].Type != eventlog.TypeFired {
							t.Fatalf("restart/cleanup invented durable lifecycle events: %+v", events)
						}
					}

					// A different host with the same container ID cannot inherit age.
					otherHost := host
					otherHost.ID, otherHost.AgentID, otherHost.Hostname = "other-host", "other-agent", "other-host"
					restarted.CheckDockerHost(otherHost)
					if got := restarted.GetActiveAlerts(); len(got) != 1 {
						t.Fatalf("other host inherited an old occurrence: %+v", got)
					}

					// Explicit recovery retires age; another pending update must wait
					// its normal delay rather than reuse the resolved occurrence.
					host.Containers[0].UpdateStatus = &models.DockerContainerUpdateStatus{CurrentDigest: "sha256:new", LatestDigest: "sha256:new", LastChecked: time.Now()}
					restarted.CheckDockerHost(host)
					if len(restarted.GetActiveAlerts()) != 0 || resolutions.Load() != 1 {
						t.Fatal("affirmative recovery did not resolve exactly once")
					}
					host.Containers[0].UpdateStatus = pending
					restarted.CheckDockerHost(host)
					if len(restarted.GetActiveAlerts()) != 0 || firings.Load() != 0 || deliveries.Load() != 0 {
						t.Fatal("new update skipped its delay or reused a resolved occurrence")
					}
					restarted.mu.RLock()
					newFirst := restarted.dockerUpdateFirstSeenByIdentity[trackingKey]
					restarted.mu.RUnlock()
					if newFirst.Before(observedAfter) {
						t.Fatalf("new update inherited old pending age: %s", newFirst)
					}
					t.Log("positive/unknown/error reports, both cleanup paths, isolated host and affirmative recovery preserved restart lifecycle")
				})
			}
		}
	}
}