Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

60.3 KB
Raw Blame
package monitoring

import (
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"slices"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
	"github.com/rcourtman/pulse-go-rewrite/internal/alerts/eventlog"
	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/mock"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host"
)

func newHostRemovalLifecycleMonitor(t *testing.T, dataPath string) *Monitor {
	t.Helper()
	events, err := eventlog.OpenInMemory()
	if err != nil {
		t.Fatalf("open alert event log: %v", err)
	}
	monitor := &Monitor{
		state:               models.NewState(),
		alertManager:        alerts.NewManagerWithDataDir(dataPath),
		hostTokenBindings:   make(map[string]string),
		removedHostAgents:   make(map[string]time.Time),
		rateTracker:         NewRateTracker(),
		config:              &config.Config{DataPath: dataPath},
		hostContinuityStore: config.NewHostContinuityStore(dataPath, nil),
	}
	// Production monitor construction enables the alerts-owned event log.
	// Exercise agent removal and re-enrollment with the same adjacent runtime
	// enabled so it cannot accidentally become lifecycle authority.
	monitor.alertManager.SetEventLog(events)
	t.Cleanup(func() { monitor.alertManager.Stop() })
	return monitor
}

func hostRemovalLifecycleReport(hostID, machineID, agentID, hostname, platform string, at time.Time) agentshost.Report {
	return agentshost.Report{
		Host: agentshost.HostInfo{
			ID:        hostID,
			MachineID: machineID,
			Hostname:  hostname,
			Platform:  platform,
		},
		Agent: agentshost.AgentInfo{
			ID:      agentID,
			Version: "6.1.1",
			Type:    "unified",
		},
		Timestamp: at.UTC(),
	}
}

func TestHostAgentRemovalLifecycleSameProcessAndPlatformAliases(t *testing.T) {
	identities := []struct {
		name      string
		hostID    string
		machineID string
		agentID   string
		hostname  string
		platform  string
	}{
		{
			name:      "linux systemd machine id",
			hostID:    "01234567-89ab-cdef-0123-456789abcdef",
			machineID: "01234567-89ab-cdef-0123-456789abcdef",
			agentID:   "linux-agent-state-id",
			hostname:  "pve-node.local",
			platform:  "linux",
		},
		{
			name:      "docker unified identity",
			hostID:    "docker-machine-7",
			machineID: "docker-machine-7",
			agentID:   "docker-agent-state-id",
			hostname:  "docker-node.local",
			platform:  "linux",
		},
		{
			name:      "windows machine guid",
			hostID:    "a12b34c5-d678-49ef-a012-3456789abcde",
			machineID: "A12B34C5-D678-49EF-A012-3456789ABCDE",
			agentID:   "windows-agent-state-id",
			hostname:  "win-node.corp.local",
			platform:  "windows",
		},
	}

	for _, identity := range identities {
		t.Run(identity.name, func(t *testing.T) {
			monitor := newHostRemovalLifecycleMonitor(t, t.TempDir())
			now := time.Now().UTC()
			oldToken := &config.APITokenRecord{
				ID:        "old-" + identity.agentID,
				CreatedAt: now.Add(-time.Hour),
			}
			report := hostRemovalLifecycleReport(
				identity.hostID,
				identity.machineID,
				identity.agentID,
				identity.hostname,
				identity.platform,
				now,
			)
			host, err := monitor.ApplyHostReport(report, oldToken)
			if err != nil {
				t.Fatalf("initial ApplyHostReport: %v", err)
			}
			if _, err := monitor.RemoveHostAgent(host.ID); err != nil {
				t.Fatalf("RemoveHostAgent: %v", err)
			}
			tombstones := monitor.hostContinuityStore.RemovedEntries()
			if len(tombstones) != 1 ||
				len(tombstones[0].DeniedTokenIDs) != 1 ||
				tombstones[0].DeniedTokenIDs[0] != oldToken.ID {
				t.Fatalf("removal tombstone token lineage = %+v", tombstones)
			}
			if _, err := monitor.ApplyHostReport(report, oldToken); err == nil {
				t.Fatal("pre-removal token re-enrolled a deliberately removed host")
			}
			if _, ok := monitor.MatchHostConfigContinuity(host.ID, oldToken.ID); ok {
				t.Fatal("removed host remained available through remote-config continuity")
			}

			aliasReport := report
			aliasReport.Host.ID = identity.agentID
			aliasReport.Timestamp = now.Add(2 * time.Minute)
			freshToken := &config.APITokenRecord{
				ID:        "fresh-" + identity.agentID,
				CreatedAt: now.Add(time.Minute),
			}
			reEnrolled, err := monitor.ApplyHostReport(aliasReport, freshToken)
			if err != nil {
				t.Fatalf("fresh-token alias re-enrollment: %v", err)
			}
			if reEnrolled.ID != host.ID {
				t.Fatalf("re-enrolled host ID = %q, want canonical %q", reEnrolled.ID, host.ID)
			}
			if got := monitor.hostContinuityStore.RemovedEntries(); len(got) != 0 {
				t.Fatalf("durable tombstone survived re-enrollment: %+v", got)
			}
			if continuity, ok := monitor.MatchHostConfigContinuity(host.ID, freshToken.ID); !ok || continuity.ID != host.ID {
				t.Fatalf("fresh-token remote-config continuity = (%+v, %v)", continuity, ok)
			}
		})
	}
}

func TestHostAgentRemovalLifecycleRevokesDedicatedCredentialAndRetainsDenial(t *testing.T) {
	dataPath := t.TempDir()
	monitor := newHostRemovalLifecycleMonitor(t, dataPath)
	monitor.persistence = config.NewConfigPersistence(dataPath)

	now := time.Now().UTC()
	oldToken := config.APITokenRecord{
		ID:        "dedicated-old-token",
		Name:      "Dedicated host token",
		CreatedAt: now.Add(-time.Hour),
	}
	monitor.config.APITokens = []config.APITokenRecord{oldToken}
	if err := monitor.persistence.SaveAPITokens(monitor.config.APITokens); err != nil {
		t.Fatalf("SaveAPITokens: %v", err)
	}

	report := hostRemovalLifecycleReport(
		"dedicated-machine-id",
		"dedicated-machine-id",
		"dedicated-agent-id",
		"dedicated.local",
		"linux",
		now,
	)
	host, err := monitor.ApplyHostReport(report, &oldToken)
	if err != nil {
		t.Fatalf("initial ApplyHostReport: %v", err)
	}
	if _, err := monitor.RemoveHostAgent(host.ID); err != nil {
		t.Fatalf("RemoveHostAgent: %v", err)
	}

	if len(monitor.config.APITokens) != 0 {
		t.Fatalf("dedicated credential remained in memory: %+v", monitor.config.APITokens)
	}
	reloadedTokens, err := monitor.persistence.LoadAPITokens()
	if err != nil {
		t.Fatalf("LoadAPITokens: %v", err)
	}
	if len(reloadedTokens) != 0 {
		t.Fatalf("dedicated credential remained on disk: %+v", reloadedTokens)
	}
	if _, err := monitor.ApplyHostReport(report, &oldToken); err == nil {
		t.Fatal("explicitly revoked credential bypassed the durable identity denial")
	}
}

func TestCollectorUninstallHostAgentTransactionRollsBackAndRetries(t *testing.T) {
	dataPath := t.TempDir()
	monitor := newHostRemovalLifecycleMonitor(t, dataPath)
	monitor.persistence = config.NewConfigPersistence(dataPath)

	now := time.Now().UTC()
	token := config.APITokenRecord{
		ID:        "collector-uninstall-token",
		Name:      "Collector uninstall token",
		Hash:      "collector-uninstall-hash",
		CreatedAt: now.Add(-time.Hour),
		Scopes:    []string{config.ScopeAgentReport, config.ScopeAgentConfigRead},
	}
	monitor.config.APITokens = []config.APITokenRecord{token}
	if err := monitor.persistence.SaveAPITokens(monitor.config.APITokens); err != nil {
		t.Fatalf("SaveAPITokens: %v", err)
	}

	report := hostRemovalLifecycleReport(
		"collector-uninstall-machine",
		"collector-uninstall-machine",
		"collector-uninstall-agent",
		"collector-uninstall.local",
		"linux",
		now,
	)
	host, err := monitor.ApplyHostReport(report, &token)
	if err != nil {
		t.Fatalf("initial ApplyHostReport: %v", err)
	}

	// Block the final replacement, not a temporary filename: credential writes
	// now own an unpredictable temporary file. Retain the original inventory
	// so recovery never needs to reconstruct it from the failed transaction.
	// The separate continuity file remains writable, exercising rollback after
	// the removal tombstone has been persisted rather than a preflight error.
	credentialBlocker := filepath.Join(dataPath, "api_tokens.json")
	retainedCredentials := filepath.Join(dataPath, "api_tokens.fixture-retained")
	if err := os.Rename(credentialBlocker, retainedCredentials); err != nil {
		t.Fatalf("retain original credential inventory: %v", err)
	}
	if err := os.Mkdir(credentialBlocker, 0o700); err != nil {
		t.Fatalf("create credential persistence blocker: %v", err)
	}
	_, uninstallErr := monitor.UninstallHostAgent(host.ID, token.ID)
	var renameErr *os.LinkError
	if !errors.As(uninstallErr, &renameErr) || renameErr.Op != "rename" || renameErr.New != credentialBlocker {
		t.Fatalf("UninstallHostAgent error = %v, want credential publication rename failure", uninstallErr)
	}
	if leftovers, err := filepath.Glob(filepath.Join(dataPath, ".api_tokens.json.*.tmp")); err != nil || len(leftovers) != 0 {
		t.Fatalf("failed publication left temporary credentials: %v (glob error: %v)", leftovers, err)
	}
	if hosts := monitor.GetLiveHostsSnapshot(); len(hosts) != 1 || hosts[0].ID != host.ID {
		t.Fatalf("failed uninstall changed live host state: %+v", hosts)
	}
	if len(monitor.config.APITokens) != 1 || monitor.config.APITokens[0].ID != token.ID {
		t.Fatalf("failed uninstall changed live credential inventory: %+v", monitor.config.APITokens)
	}
	continuity, ok := monitor.hostContinuityStore.Get(host.ID)
	if !ok || !continuity.RemovedAt.IsZero() || continuity.TokenID != token.ID {
		t.Fatalf("failed uninstall did not restore active continuity: (%+v, %v)", continuity, ok)
	}

	if err := os.Remove(credentialBlocker); err != nil {
		t.Fatalf("remove credential persistence blocker: %v", err)
	}
	if err := os.Rename(retainedCredentials, credentialBlocker); err != nil {
		t.Fatalf("restore original credential inventory: %v", err)
	}
	persistedBeforeRetry, err := monitor.persistence.LoadAPITokens()
	if err != nil || len(persistedBeforeRetry) != 1 || persistedBeforeRetry[0].ID != token.ID {
		t.Fatalf("failed uninstall did not retain durable retry credential: %+v (error: %v)", persistedBeforeRetry, err)
	}
	removed, err := monitor.UninstallHostAgent(host.ID, token.ID)
	if err != nil {
		t.Fatalf("retry UninstallHostAgent: %v", err)
	}
	if removed.ID != host.ID {
		t.Fatalf("removed host = %+v, want %q", removed, host.ID)
	}
	if hosts := monitor.GetLiveHostsSnapshot(); len(hosts) != 0 {
		t.Fatalf("successful uninstall retained live host: %+v", hosts)
	}
	if len(monitor.config.APITokens) != 0 {
		t.Fatalf("successful uninstall retained live credential: %+v", monitor.config.APITokens)
	}
	persistedTokens, err := monitor.persistence.LoadAPITokens()
	if err != nil {
		t.Fatalf("LoadAPITokens after retry: %v", err)
	}
	if len(persistedTokens) != 0 {
		t.Fatalf("successful uninstall retained durable credential: %+v", persistedTokens)
	}
	tombstone, ok := monitor.hostContinuityStore.Get(host.ID)
	if !ok || tombstone.RemovedAt.IsZero() || !slices.Contains(tombstone.DeniedTokenIDs, token.ID) {
		t.Fatalf("successful uninstall tombstone = (%+v, %v)", tombstone, ok)
	}
	if _, err := monitor.UninstallHostAgent(host.ID, token.ID); err != nil {
		t.Fatalf("idempotent uninstall retry: %v", err)
	}
}

func TestRevokeAPITokenRollsBackCompleteInventoryWhenPersistenceFails(t *testing.T) {
	now := time.Now().UTC()
	tokens := []config.APITokenRecord{
		{ID: "newest", Name: "newest", Hash: "hash-newest", CreatedAt: now, Scopes: []string{config.ScopeWildcard}},
		{ID: "target", Name: "target", Hash: "hash-target", CreatedAt: now.Add(-time.Minute), Scopes: []string{config.ScopeAgentReport}},
		{ID: "oldest", Name: "oldest", Hash: "hash-oldest", CreatedAt: now.Add(-2 * time.Minute), Scopes: []string{config.ScopeMonitoringRead}},
	}
	stateDir := filepath.Join(t.TempDir(), "state")
	persistence := config.NewConfigPersistence(stateDir)
	if err := persistence.SaveAPITokens(tokens); err != nil {
		t.Fatalf("save initial tokens: %v", err)
	}
	if err := os.RemoveAll(stateDir); err != nil {
		t.Fatalf("remove persistence directory: %v", err)
	}
	if err := os.WriteFile(stateDir, []byte("not a directory"), 0o600); err != nil {
		t.Fatalf("create persistence blocker: %v", err)
	}

	monitor := &Monitor{
		config:      &config.Config{APITokens: append([]config.APITokenRecord(nil), tokens...)},
		persistence: persistence,
	}
	monitor.config.SortAPITokens()

	removed, err := monitor.revokeAPIToken("target")
	if err == nil {
		t.Fatal("revokeAPIToken succeeded despite persistence failure")
	}
	if removed != nil {
		t.Fatalf("failed revocation returned removed token: %#v", removed)
	}
	if len(monitor.config.APITokens) != len(tokens) {
		t.Fatalf("token count = %d, want %d: %#v", len(monitor.config.APITokens), len(tokens), monitor.config.APITokens)
	}
	for index, want := range tokens {
		if monitor.config.APITokens[index].ID != want.ID {
			t.Fatalf("token[%d].ID = %q, want %q", index, monitor.config.APITokens[index].ID, want.ID)
		}
	}
	if monitor.config.APIToken != "hash-newest" {
		t.Fatalf("legacy primary token = %q, want %q", monitor.config.APIToken, "hash-newest")
	}
}

func TestRevokeAPITokenCommitsExactReducedInventory(t *testing.T) {
	stateDir := t.TempDir()
	persistence := config.NewConfigPersistence(stateDir)
	monitor := &Monitor{
		config: &config.Config{APITokens: []config.APITokenRecord{
			{ID: "keep", Name: "keep", Hash: "hash-keep", CreatedAt: time.Now().UTC(), Scopes: []string{config.ScopeWildcard}},
			{ID: "remove", Name: "remove", Hash: "hash-remove", CreatedAt: time.Now().UTC().Add(-time.Minute), Scopes: []string{config.ScopeAgentReport}},
		}},
		persistence: persistence,
	}
	monitor.config.SortAPITokens()

	removed, err := monitor.revokeAPIToken("remove")
	if err != nil {
		t.Fatalf("revokeAPIToken: %v", err)
	}
	if removed == nil || removed.ID != "remove" {
		t.Fatalf("removed token = %#v", removed)
	}
	if len(monitor.config.APITokens) != 1 || monitor.config.APITokens[0].ID != "keep" {
		t.Fatalf("live token inventory = %#v", monitor.config.APITokens)
	}
	persisted, err := persistence.LoadAPITokens()
	if err != nil {
		t.Fatalf("LoadAPITokens: %v", err)
	}
	if len(persisted) != 1 || persisted[0].ID != "keep" {
		t.Fatalf("persisted token inventory = %#v", persisted)
	}
}

func TestHostAgentRemovalLifecycleRepublishesUnifiedReadState(t *testing.T) {
	monitor := newHostRemovalLifecycleMonitor(t, t.TempDir())
	adapter := unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(nil))
	monitor.SetResourceStore(adapter)

	now := time.Now().UTC()
	report := hostRemovalLifecycleReport(
		"state-convergence-machine",
		"state-convergence-machine",
		"state-convergence-agent",
		"state-convergence.local",
		"linux",
		now,
	)
	host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{
		ID:        "state-convergence-token",
		CreatedAt: now.Add(-time.Hour),
	})
	if err != nil {
		t.Fatalf("ApplyHostReport: %v", err)
	}
	if hosts := adapter.Hosts(); len(hosts) != 1 {
		t.Fatalf("unified read state before removal = %+v, want one host", hosts)
	}

	if _, err := monitor.RemoveHostAgent(host.ID); err != nil {
		t.Fatalf("RemoveHostAgent: %v", err)
	}
	if hosts := adapter.Hosts(); len(hosts) != 0 {
		t.Fatalf("unified read state retained removed host: %+v", hosts)
	}
}

func TestHostAgentRemovalLifecycleRemovesAssociatedDockerSurfacesAndAlerts(t *testing.T) {
	monitor := newHostRemovalLifecycleMonitor(t, t.TempDir())
	now := time.Now().UTC()
	report := hostRemovalLifecycleReport(
		"host-machine-id",
		"host-machine-id",
		"host-agent-id",
		"shared-name.local",
		"linux",
		now,
	)
	host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{
		ID:        "host-token",
		CreatedAt: now.Add(-time.Hour),
	})
	if err != nil {
		t.Fatalf("ApplyHostReport: %v", err)
	}

	dockerHosts := []models.DockerHost{
		{
			ID:          "docker-agent-alias",
			AgentID:     report.Agent.ID,
			Hostname:    report.Host.Hostname,
			DisplayName: report.Host.Hostname,
			Status:      "offline",
			TokenID:     "docker-token-one",
		},
		{
			ID:          "docker-machine-alias",
			MachineID:   report.Host.MachineID,
			Hostname:    report.Host.Hostname,
			DisplayName: report.Host.Hostname,
			Status:      "offline",
			TokenID:     "docker-token-two",
		},
		{
			ID:          "unrelated-docker-host",
			AgentID:     "unrelated-agent",
			MachineID:   "unrelated-machine",
			Hostname:    report.Host.Hostname,
			DisplayName: report.Host.Hostname,
			Status:      "offline",
			TokenID:     "unrelated-token",
		},
	}
	for _, dockerHost := range dockerHosts {
		monitor.state.UpsertDockerHost(dockerHost)
		for range 3 {
			monitor.alertManager.HandleDockerHostOffline(dockerHost)
		}
	}
	activeResources := make(map[string]bool)
	for _, alert := range monitor.alertManager.GetActiveAlerts() {
		activeResources[alert.ResourceID] = true
	}
	for _, dockerHost := range dockerHosts {
		resourceID := "docker:" + dockerHost.ID
		if !activeResources[resourceID] {
			t.Fatalf("active alerts before host removal did not include %q", resourceID)
		}
	}

	if _, err := monitor.RemoveHostAgent(host.ID); err != nil {
		t.Fatalf("RemoveHostAgent: %v", err)
	}
	remaining := monitor.state.GetDockerHosts()
	if len(remaining) != 1 || remaining[0].ID != "unrelated-docker-host" {
		t.Fatalf("remaining Docker hosts = %+v, want only unrelated host", remaining)
	}
	activeResources = make(map[string]bool)
	for _, alert := range monitor.alertManager.GetActiveAlerts() {
		activeResources[alert.ResourceID] = true
	}
	if activeResources["docker:docker-agent-alias"] || activeResources["docker:docker-machine-alias"] {
		t.Fatalf("associated Docker alerts remained active after host removal: %+v", activeResources)
	}
	if !activeResources["docker:unrelated-docker-host"] {
		t.Fatalf("unrelated Docker alert was cleared after host removal: %+v", activeResources)
	}

	removedIDs := make(map[string]bool)
	for _, removed := range monitor.state.GetRemovedDockerHosts() {
		removedIDs[removed.ID] = true
	}
	if !removedIDs["docker-agent-alias"] || !removedIDs["docker-machine-alias"] {
		t.Fatalf("removed Docker hosts = %+v, want both associated surfaces", removedIDs)
	}
	if removedIDs["unrelated-docker-host"] {
		t.Fatal("same-hostname unrelated Docker host was removed")
	}
}

func TestHostAgentRemovalLifecycleSurvivesMonitorReconstruction(t *testing.T) {
	dataPath := t.TempDir()
	now := time.Now().UTC()
	oldToken := &config.APITokenRecord{
		ID:        "shared-old-token",
		CreatedAt: now.Add(-time.Hour),
	}
	report := hostRemovalLifecycleReport(
		"systemd-machine-id",
		"systemd-machine-id",
		"persisted-agent-id",
		"restart-node.local",
		"linux",
		now,
	)

	first := newHostRemovalLifecycleMonitor(t, dataPath)
	host, err := first.ApplyHostReport(report, oldToken)
	if err != nil {
		t.Fatalf("initial ApplyHostReport: %v", err)
	}
	if _, err := first.RemoveHostAgent(host.ID); err != nil {
		t.Fatalf("RemoveHostAgent: %v", err)
	}

	restarted, err := New(&config.Config{
		DataPath:   dataPath,
		ConfigPath: dataPath,
		MetricsDBPath: filepath.Join(
			dataPath,
			"restarted-metrics.db",
		),
	})
	if err != nil {
		t.Fatalf("New restarted monitor: %v", err)
	}
	t.Cleanup(restarted.Stop)

	aliasReport := report
	aliasReport.Host.ID = report.Agent.ID
	aliasReport.Timestamp = now.Add(2 * time.Minute)
	if _, err := restarted.ApplyHostReport(aliasReport, oldToken); err == nil {
		t.Fatal("restart lost the durable removal deny boundary")
	}
	if _, ok := restarted.MatchHostConfigContinuity(host.ID, oldToken.ID); ok {
		t.Fatal("restart exposed removed host through remote-config continuity")
	}

	wrongIdentity := aliasReport
	wrongIdentity.Host.MachineID = "cloned-machine-id"
	wrongIdentity.Host.Hostname = "different-node.local"
	wrongIdentity.Timestamp = now.Add(3 * time.Minute)
	freshToken := &config.APITokenRecord{
		ID:        "fresh-after-restart",
		CreatedAt: now.Add(time.Minute),
	}
	if _, err := restarted.ApplyHostReport(wrongIdentity, freshToken); err == nil {
		t.Fatal("fresh token cleared a tombstone for a different machine identity")
	}

	reEnrolled, err := restarted.ApplyHostReport(aliasReport, freshToken)
	if err != nil {
		t.Fatalf("fresh-token re-enrollment after restart: %v", err)
	}
	if reEnrolled.ID != host.ID {
		t.Fatalf("re-enrolled host ID = %q, want canonical %q", reEnrolled.ID, host.ID)
	}
	aliasReport.Timestamp = now.Add(4 * time.Minute)
	if _, err := restarted.ApplyHostReport(aliasReport, oldToken); err == nil {
		t.Fatal("detached old token created a duplicate after fresh-token re-enrollment")
	}
	if got := restarted.GetLiveHostsSnapshot(); len(got) != 1 || got[0].ID != host.ID {
		t.Fatalf("detached old token changed active host inventory: %+v", got)
	}
}

func TestHostAgentRemovalLifecycleKeepsOfflineRowRemovableAfterRestart(t *testing.T) {
	dataPath := t.TempDir()
	now := time.Now().UTC()
	token := &config.APITokenRecord{
		ID:        "offline-restart-token",
		CreatedAt: now.Add(-time.Hour),
	}
	report := hostRemovalLifecycleReport(
		"offline-restart-machine",
		"offline-restart-machine",
		"offline-restart-agent",
		"offline-restart.local",
		"linux",
		now,
	)

	first := newHostRemovalLifecycleMonitor(t, dataPath)
	host, err := first.ApplyHostReport(report, token)
	if err != nil {
		t.Fatalf("initial ApplyHostReport: %v", err)
	}

	// Reconstruct the monitor without another agent report. The durable row
	// must remain visible while awaiting a report so removal stays reachable.
	restarted := newHostRemovalLifecycleMonitor(t, dataPath)
	hosts := restarted.HostsSnapshot()
	if len(hosts) != 1 || hosts[0].ID != host.ID {
		t.Fatalf("restart host inventory = %+v, want continuity row %q", hosts, host.ID)
	}
	if hosts[0].Status != "pending" {
		t.Fatalf("restart continuity status = %q, want pending", hosts[0].Status)
	}

	for i := 0; i < 3; i++ {
		restarted.evaluateHostAgents(now.Add(10*time.Minute + time.Duration(i)*time.Second))
	}
	if alerts := restarted.alertManager.GetActiveAlerts(); len(alerts) != 1 || alerts[0].Type != "host-offline" {
		t.Fatalf("restart offline alerts = %+v, want one host-offline alert", alerts)
	}

	removed, err := restarted.RemoveHostAgent(host.ID)
	if err != nil {
		t.Fatalf("RemoveHostAgent after restart: %v", err)
	}
	if removed.ID != host.ID || removed.Hostname != host.Hostname {
		t.Fatalf("removed continuity host = %+v, want %+v", removed, host)
	}
	if hosts := restarted.HostsSnapshot(); len(hosts) != 0 {
		t.Fatalf("removed host remained visible after restart: %+v", hosts)
	}
	if alerts := restarted.alertManager.GetActiveAlerts(); len(alerts) != 0 {
		t.Fatalf("removed host alerts remained active: %+v", alerts)
	}
	if tombstones := restarted.hostContinuityStore.RemovedEntries(); len(tombstones) != 1 || tombstones[0].HostID != host.ID {
		t.Fatalf("removal tombstones = %+v, want host %q", tombstones, host.ID)
	}
}

// After a restart a host that has not reported again exists only as its saved
// continuity row, and its offline alert names it agent:<host ID>. The intent
// the operator set on that row must still hold the alert back: alert intent
// resolves through the read state that overlays saved hosts, not through the
// published registry alone, which no longer lists the host.
func TestHostAgentRemovalLifecycleHonorsSavedHostIntentAfterRestart(t *testing.T) {
	dataPath := t.TempDir()
	now := time.Now().UTC()
	token := &config.APITokenRecord{ID: "intent-restart-token", CreatedAt: now.Add(-time.Hour)}
	report := hostRemovalLifecycleReport("intent-restart-host", "intent-restart-machine", "intent-restart-agent", "intent-restart.local", "linux", now)
	if _, err := newHostRemovalLifecycleMonitor(t, dataPath).ApplyHostReport(report, token); err != nil {
		t.Fatalf("initial ApplyHostReport: %v", err)
	}

	restarted := newHostRemovalLifecycleMonitor(t, dataPath)
	store := unifiedresources.NewMemoryStore()
	savedID := unifiedresources.MachineIdentityCanonicalID(unifiedresources.ResourceTypeAgent, "intent-restart-machine")
	if err := store.SetResourceOperatorState(unifiedresources.ResourceOperatorState{CanonicalID: savedID, IntentionallyOffline: true}); err != nil {
		t.Fatalf("set saved host intent: %v", err)
	}
	restarted.SetResourceStore(unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)))
	for i := 0; i < 3; i++ {
		restarted.evaluateHostAgents(now.Add(10*time.Minute + time.Duration(i)*time.Second))
	}
	if alerts := restarted.alertManager.GetActiveAlerts(); len(alerts) != 0 {
		t.Fatalf("intentionally offline saved host raised %+v after restart, want no alert", alerts)
	}
}

func TestHostAgentRemovalLifecycleDoesNotPoisonDuplicateActiveIdentity(t *testing.T) {
	monitor := newHostRemovalLifecycleMonitor(t, t.TempDir())
	now := time.Now().UTC()
	firstReport := hostRemovalLifecycleReport("shared-machine", "shared-machine", "agent-one", "node-one.local", "linux", now)
	secondReport := hostRemovalLifecycleReport("shared-machine", "shared-machine", "agent-two", "node-two.local", "linux", now)
	firstToken := &config.APITokenRecord{ID: "token-one", CreatedAt: now.Add(-time.Hour)}
	secondToken := &config.APITokenRecord{ID: "token-two", CreatedAt: now.Add(-time.Hour)}

	first, err := monitor.ApplyHostReport(firstReport, firstToken)
	if err != nil {
		t.Fatalf("first ApplyHostReport: %v", err)
	}
	second, err := monitor.ApplyHostReport(secondReport, secondToken)
	if err != nil {
		t.Fatalf("second ApplyHostReport: %v", err)
	}
	if first.ID == second.ID {
		t.Fatalf("duplicate active identities collapsed to %q", first.ID)
	}

	if _, err := monitor.RemoveHostAgent(first.ID); err != nil {
		t.Fatalf("RemoveHostAgent(first): %v", err)
	}
	secondReport.Timestamp = now.Add(time.Minute)
	updated, err := monitor.ApplyHostReport(secondReport, secondToken)
	if err != nil {
		t.Fatalf("unrelated duplicate host was poisoned by removal: %v", err)
	}
	if updated.ID != second.ID {
		t.Fatalf("unrelated duplicate host changed ID from %q to %q", second.ID, updated.ID)
	}
}

func TestHostAgentRemovalLifecycleDoesNotUseProxmoxDisplayNameAsIdentity(t *testing.T) {
	monitor := newHostRemovalLifecycleMonitor(t, t.TempDir())
	monitor.state.UpdateNodesForInstance("production-api", []models.Node{{
		ID:              "production-pve1",
		NodeIdentity:    "production-pve1",
		Name:            "pve1",
		DisplayName:     "Render East",
		Instance:        "production-api",
		IsClusterMember: true,
		LinkedAgentID:   "agent-machine",
	}})

	now := time.Now().UTC()
	report := hostRemovalLifecycleReport(
		"agent-machine",
		"agent-machine",
		"agent-state",
		"pve1",
		"linux",
		now,
	)
	token := &config.APITokenRecord{ID: "agent-token", CreatedAt: now.Add(-time.Hour)}
	host, err := monitor.ApplyHostReport(report, token)
	if err != nil {
		t.Fatalf("ApplyHostReport: %v", err)
	}
	if _, err := monitor.RemoveHostAgent(host.ID); err != nil {
		t.Fatalf("RemoveHostAgent: %v", err)
	}

	nodes := monitor.state.GetSnapshot().Nodes
	if len(nodes) != 1 ||
		nodes[0].NodeIdentity != "production-pve1" ||
		nodes[0].Name != "pve1" ||
		nodes[0].DisplayName != "Render East" {
		t.Fatalf("agent removal changed provider-owned node presentation identity: %+v", nodes)
	}
}

func TestHostAgentRemovalLifecycleFailsClosedWhenTombstoneCannotPersist(t *testing.T) {
	dataPath := filepath.Join(t.TempDir(), "not-a-directory")
	if err := os.WriteFile(dataPath, []byte("fixture"), 0o600); err != nil {
		t.Fatalf("write non-directory fixture: %v", err)
	}
	monitor := newHostRemovalLifecycleMonitor(t, dataPath)
	now := time.Now().UTC()
	report := hostRemovalLifecycleReport("persist-failure-host", "persist-failure-host", "agent-id", "persist.local", "linux", now)
	host, err := monitor.ApplyHostReport(report, &config.APITokenRecord{ID: "persist-token", CreatedAt: now.Add(-time.Hour)})
	if err != nil {
		t.Fatalf("ApplyHostReport: %v", err)
	}

	if _, err := monitor.RemoveHostAgent(host.ID); err == nil {
		t.Fatal("RemoveHostAgent succeeded without a durable tombstone")
	}
	if got := monitor.state.GetHosts(); len(got) != 1 || got[0].ID != host.ID {
		t.Fatalf("failed removal did not restore live host: %+v", got)
	}
}

func TestHostAgentRemovalLifecycleFailsClosedWhenJournalCannotLoad(t *testing.T) {
	dataPath := t.TempDir()
	if err := os.WriteFile(
		filepath.Join(dataPath, "host_continuity.json"),
		[]byte(`{"broken":`),
		0o600,
	); err != nil {
		t.Fatalf("write corrupt journal: %v", err)
	}
	if monitor, err := New(&config.Config{DataPath: dataPath, ConfigPath: dataPath}); err == nil {
		monitor.Stop()
		t.Fatal("monitor started without a readable host lifecycle journal")
	}
}

func TestHostAgentRemovalLifecycleOrdersConcurrentReportsBeforeDeletion(t *testing.T) {
	monitor := newHostRemovalLifecycleMonitor(t, t.TempDir())
	now := time.Now().UTC()
	report := hostRemovalLifecycleReport("race-machine", "race-machine", "race-agent", "race.local", "linux", now)
	token := &config.APITokenRecord{ID: "race-token", CreatedAt: now.Add(-time.Hour)}
	host, err := monitor.ApplyHostReport(report, token)
	if err != nil {
		t.Fatalf("initial ApplyHostReport: %v", err)
	}

	const reporters = 32
	start := make(chan struct{})
	var wg sync.WaitGroup
	wg.Add(reporters)
	for i := 0; i < reporters; i++ {
		go func(offset int) {
			defer wg.Done()
			<-start
			concurrent := report
			concurrent.Timestamp = now.Add(time.Duration(offset+1) * time.Millisecond)
			_, _ = monitor.ApplyHostReport(concurrent, token)
		}(i)
	}
	close(start)
	if _, err := monitor.RemoveHostAgent(host.ID); err != nil {
		t.Fatalf("RemoveHostAgent: %v", err)
	}
	wg.Wait()

	if got := monitor.state.GetHosts(); len(got) != 0 {
		t.Fatalf("concurrent report resurrected removed host: %+v", got)
	}
	report.Timestamp = now.Add(time.Minute)
	if _, err := monitor.ApplyHostReport(report, token); err == nil {
		t.Fatal("old token reported after concurrent deletion completed")
	}
}

func TestUnifiedStorageMetricSyncPreservesHostRemovalBlock(t *testing.T) {
	monitor := newHostRemovalLifecycleMonitor(t, t.TempDir())
	now := time.Now().UTC().Truncate(time.Second)
	report := hostRemovalLifecycleReport(
		"storage-sync-machine",
		"storage-sync-machine",
		"storage-sync-agent",
		"storage-sync.local",
		"linux",
		now,
	)
	token := &config.APITokenRecord{ID: "storage-sync-token", CreatedAt: now.Add(-time.Hour)}
	host, err := monitor.ApplyHostReport(report, token)
	if err != nil {
		t.Fatalf("initial ApplyHostReport: %v", err)
	}
	if _, err := monitor.RemoveHostAgent(host.ID); err != nil {
		t.Fatalf("RemoveHostAgent: %v", err)
	}

	resourceStore := unifiedresources.NewMonitorAdapter(nil)
	resourceStore.PopulateFromSnapshot(models.StateSnapshot{
		PBSInstances: []models.PBSInstance{{
			ID:       "pbs-lifecycle-boundary",
			Name:     "pbs-lifecycle-boundary",
			Status:   "online",
			LastSeen: now,
			Datastores: []models.PBSDatastore{{
				Name:   "backups",
				Status: "available",
				Total:  1000,
				Used:   400,
				Free:   600,
				Usage:  40,
			}},
		}},
	})
	monitor.metricsHistory = NewMetricsHistory(16, time.Hour)
	monitor.syncUnifiedStorageMetrics(resourceStore)

	if got := monitor.hostContinuityStore.RemovedEntries(); len(got) != 1 {
		t.Fatalf("storage metric sync changed durable removal entries: %+v", got)
	}
	report.Timestamp = now.Add(time.Minute)
	if _, err := monitor.ApplyHostReport(report, token); err == nil {
		t.Fatal("storage metric sync allowed a removed host to report with its denied token")
	}
}

func TestMockHostAgentLeavingFixtureUsesRemovalLifecycle(t *testing.T) {
	mustSetMockEnabled(t, true)
	t.Cleanup(func() { mustSetMockEnabled(t, false) })
	manager := newMockHostAlertTestManager(t)
	monitor := &Monitor{alertManager: manager}

	memory := models.Memory{Total: 32 << 30, Used: 30 << 30, Free: 2 << 30, Usage: 93.75}
	kept := models.Host{ID: "host-linux-1", Hostname: "apollo-114", Status: "online", Memory: memory}
	departed := models.Host{ID: "host-node-pve9", Hostname: "pve9", LinkedNodeID: "mock-cluster-1-pve9", Status: "online", Memory: memory}

	monitor.evaluateMockHostAgents(monitor.mockModeFence.begin(), []models.Host{kept, departed}, nil, 1)
	// A runtime mock config change rebuilt the estate without pve9.
	monitor.evaluateMockHostAgents(monitor.mockModeFence.begin(), []models.Host{kept}, nil, 2)

	keptAlert := false
	for _, alert := range manager.GetActiveAlerts() {
		switch alert.ResourceID {
		case "agent:" + departed.ID:
			t.Fatalf("agent that left the mock estate kept alert %s", alert.ID)
		case "agent:" + kept.ID:
			keptAlert = keptAlert || alert.Type == "memory"
		}
	}
	if !keptAlert {
		t.Fatal("agent still in the mock estate lost its memory alert")
	}

	// The departed agent's node link is dropped too, so the node it was
	// linked to owns its metric alerts again.
	node := models.Node{
		ID:       "mock-cluster-1-pve9",
		Name:     "pve9",
		Instance: "mock-cluster-1",
		Status:   "online",
		Memory:   memory,
	}
	manager.CheckNode(node)
	for _, alert := range manager.GetActiveAlerts() {
		if alert.ResourceID == node.ID && alert.Type == "memory" {
			return
		}
	}
	t.Fatal("node pve9 raised no memory alert after its agent left the mock estate")
}

func TestMockDockerHostLeavingFixtureUsesRemovalLifecycle(t *testing.T) {
	mustSetMockEnabled(t, true)
	t.Cleanup(func() { mustSetMockEnabled(t, false) })
	manager := newMockHostAlertTestManager(t)
	monitor := &Monitor{alertManager: manager}

	kept := newMockDockerHostWithExitedContainers("nebula-1-mock", "nebula-1")
	// Nested Docker-in-LXC hosts are named after their guest's VMID, which a
	// runtime mock config change can shift.
	departed := newMockDockerHostWithExitedContainers("proxmox-lxc-docker:Production West:pve1:108", "pve1-ct108")

	monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept, departed}, 1)
	monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept, departed}, 1)
	if ids := dockerAlertIDsForHost(manager, departed.ID); len(ids) != len(departed.Containers) {
		t.Fatalf("departing host opened %v, want one alert per exited container", ids)
	}

	monitor.evaluateMockDockerHosts(monitor.mockModeFence.begin(), []models.DockerHost{kept}, 2)

	if ids := dockerAlertIDsForHost(manager, departed.ID); len(ids) != 0 {
		t.Fatalf("Docker host that left the mock estate kept alerts %v", ids)
	}
	if ids := dockerAlertIDsForHost(manager, kept.ID); len(ids) != len(kept.Containers) {
		t.Fatalf("Docker host still in the mock estate has alerts %v, want one per exited container", ids)
	}
}

func TestLeavingMockModeReleasesFixtureAgentNodeLinksOnEveryRunningMonitor(t *testing.T) {
	setMockSamplerTestEnv(t, time.Hour, 5*time.Minute)
	pinDefaultMockEstate(t)
	mustSetMockEnabled(t, true)
	defaultMonitor, tenantMonitor := startTenantMonitors(t)

	// org-b's start pass registers the fixture agents in org-b's own alert
	// manager, each owning the usage alerts of the fixture node it runs on.
	var linked models.Node
	waitForCondition(t, 30*time.Second, func() bool {
		tenantMonitor.mockHostAgentsMu.Lock()
		defer tenantMonitor.mockHostAgentsMu.Unlock()
		for _, host := range tenantMonitor.mockHostAgents {
			if host.Status != "online" || host.LinkedNodeID == "" {
				continue
			}
			for _, node := range mock.CurrentFixtureGraph().State.Nodes {
				if node.ID == host.LinkedNodeID {
					linked = node
					return true
				}
			}
		}
		return false
	}, "org-b registered no online fixture agent linked to a fixture node")

	// The demo-fixture licence sync switches through the default monitor.
	mustSetMonitorMockMode(t, defaultMonitor, false)

	// org-b's fixture agents go through the removal lifecycle too, so the
	// node an agent covered owns its metric alerts again.
	linked.Status = "online"
	linked.Memory = models.Memory{Total: 64 << 30, Used: 60 << 30, Free: 4 << 30, Usage: 93.75}
	tenantMonitor.alertManager.CheckNode(linked)
	for _, alert := range tenantMonitor.alertManager.GetActiveAlerts() {
		if alert.ResourceID == linked.ID && alert.Type == "memory" {
			return
		}
	}
	t.Fatalf("org-b's node %s raised no memory alert: its fixture agent's node link outlived a switch made through the default monitor", linked.ID)
}

// nodeAgentSplitMonitor is a Proxmox node with a guest and the pulse-agent
// on the same machine, monitored through a store-backed resource adapter.
type nodeAgentSplitMonitor struct {
	t       *testing.T
	monitor *Monitor
	store   unifiedresources.ResourceStore
	adapter *unifiedresources.MonitorAdapter
	nodes   []models.Node
	vm      models.VM
	report  agentshost.Report
	host    models.Host
}

func newNodeAgentSplitMonitor(t *testing.T) *nodeAgentSplitMonitor {
	return newNodeAgentSplitMonitorWithStore(t, nil)
}

// flakyDecisionStore fails reads of the operator's manual decisions while
// fail is set, as a store that cannot be read for a moment does.
type flakyDecisionStore struct {
	unifiedresources.ResourceStore
	fail atomic.Bool
}

func (s *flakyDecisionStore) GetLinks() ([]unifiedresources.ResourceLink, error) {
	if s.fail.Load() {
		return nil, errors.New("decisions unreadable")
	}
	return s.ResourceStore.GetLinks()
}

func (s *flakyDecisionStore) GetExclusions() ([]unifiedresources.ResourceExclusion, error) {
	if s.fail.Load() {
		return nil, errors.New("decisions unreadable")
	}
	return s.ResourceStore.GetExclusions()
}

func newNodeAgentSplitMonitorWithStore(t *testing.T, wrap func(unifiedresources.ResourceStore) unifiedresources.ResourceStore) *nodeAgentSplitMonitor {
	t.Helper()
	sqliteStore, err := unifiedresources.NewSQLiteResourceStore(t.TempDir(), "default")
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = sqliteStore.Close() })
	var store unifiedresources.ResourceStore = sqliteStore
	if wrap != nil {
		store = wrap(store)
	}
	now := time.Now().UTC()
	f := &nodeAgentSplitMonitor{
		t:       t,
		monitor: issue1654Monitor(),
		store:   store,
		adapter: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(store)),
		nodes: []models.Node{
			{ID: "lab-pve1", Name: "pve1", Instance: "lab", Host: "https://10.0.0.5:8006", Status: "online", LastSeen: now},
			{ID: "lab-pve2", Name: "pve2", Instance: "lab", Host: "https://10.0.0.6:8006", Status: "online", LastSeen: now},
		},
		vm: models.VM{ID: "lab-pve1-100", VMID: 100, Name: "web", Node: "pve1", Instance: "lab", Type: "qemu", Status: "running", LastSeen: now},
		report: agentshost.Report{
			Agent: agentshost.AgentInfo{ID: "agent-pve1", Version: "6.2.0", IntervalSeconds: 30, DiskExclude: []string{"/dev/sdz"}},
			Host: agentshost.HostInfo{
				ID: "pve1-host", MachineID: "0123456789abcdef", Hostname: "pve1", Platform: "linux", ReportIP: "10.0.0.5",
			},
			Timestamp: now,
		},
	}
	f.monitor.hostContinuityStore = config.NewHostContinuityStore(t.TempDir(), nil)
	f.monitor.SetResourceStore(f.adapter)
	return f
}

// cycle polls the nodes, rebuilds the adapter, takes one agent report and
// rebuilds again, as the poll loop and report ingest interleave.
func (f *nodeAgentSplitMonitor) cycle() {
	f.t.Helper()
	f.monitor.state.UpdateNodesForInstance("lab", f.nodes)
	f.monitor.state.UpdateVMsForInstance("lab", []models.VM{f.vm})
	f.adapter.PopulateFromSnapshot(f.monitor.state.GetSnapshot())
	f.report.Timestamp = f.report.Timestamp.Add(time.Second)
	host, err := f.monitor.ApplyHostReport(f.report, nil)
	if err != nil {
		f.t.Fatal(err)
	}
	f.host = host
	f.adapter.PopulateFromSnapshot(f.monitor.state.GetSnapshot())
}

// recordReportMerge records what POST /api/resources/{id}/report-merge
// records for the joined node and agent row, read from a registry rebuilt
// from the monitor's snapshot as the resources API builds one.
func (f *nodeAgentSplitMonitor) recordReportMerge() {
	f.t.Helper()
	registry := unifiedresources.NewRegistry(f.store)
	registry.IngestSnapshot(f.monitor.state.GetSnapshot())
	merged := ""
	for _, resource := range registry.List() {
		if resource.Proxmox != nil && resource.Agent != nil {
			merged = resource.ID
		}
	}
	if merged == "" {
		f.t.Fatal("no joined node and agent row to report")
	}
	for _, target := range registry.SourceTargets(merged) {
		if target.CandidateID == "" || target.CandidateID == merged {
			continue
		}
		if err := f.store.AddExclusion(unifiedresources.ResourceExclusion{ResourceA: merged, ResourceB: target.CandidateID, CreatedAt: time.Now().UTC()}); err != nil {
			f.t.Fatal(err)
		}
	}
}

// seedLXCFilesystems caches one container filesystem reading from the agent,
// as an earlier report with a Proxmox LXC inventory would have.
func (f *nodeAgentSplitMonitor) seedLXCFilesystems() {
	f.t.Helper()
	f.monitor.proxmoxLXCFilesystemsMu.Lock()
	defer f.monitor.proxmoxLXCFilesystemsMu.Unlock()
	f.monitor.proxmoxLXCFilesystemsCache = map[string]agentLXCFilesystemCacheEntry{
		agentLXCFilesystemCacheKey("lab", "pve1", 200): {
			agentID:   f.host.ID,
			name:      "ct200",
			disks:     []models.Disk{{Mountpoint: "/", Total: 10 << 30, Used: 1 << 30}},
			expiresAt: time.Now().Add(10 * time.Minute),
		},
	}
}

func (f *nodeAgentSplitMonitor) lxcFilesystemEntries() int {
	f.monitor.proxmoxLXCFilesystemsMu.RLock()
	defer f.monitor.proxmoxLXCFilesystemsMu.RUnlock()
	return len(f.monitor.proxmoxLXCFilesystemsCache)
}

// reportOnly takes one agent report with no poll or rebuild before it.
func (f *nodeAgentSplitMonitor) reportOnly() models.Host {
	f.t.Helper()
	f.report.Timestamp = f.report.Timestamp.Add(time.Second)
	host, err := f.monitor.ApplyHostReport(f.report, nil)
	if err != nil {
		f.t.Fatal(err)
	}
	f.host = host
	return host
}

// exclusions lists the store's exclusions as "a|b", and how many name the
// node's own source-specific ID.
func (f *nodeAgentSplitMonitor) exclusions() (all []string, namingNode int) {
	f.t.Helper()
	exclusions, err := f.store.GetExclusions()
	if err != nil {
		f.t.Fatal(err)
	}
	nodeCandidate := unifiedresources.SourceSpecificID(unifiedresources.ResourceTypeAgent, unifiedresources.SourceProxmox, f.nodes[0].ID)
	for _, exclusion := range exclusions {
		all = append(all, exclusion.ResourceA+"|"+exclusion.ResourceB)
		if exclusion.ResourceA == nodeCandidate || exclusion.ResourceB == nodeCandidate {
			namingNode++
		}
	}
	slices.Sort(all)
	return all, namingNode
}

// assertLinked checks every reader of the node<->agent link: the state's
// two sides, report ingest's result, the node's linked agent in the read
// state (agent deployment, service discovery and the physical disk poll's
// SMART fallback and --disk-exclude patterns read it), the node's disk
// source host, the guest's inherited agent (guest discovery and commands)
// and the shared-system alert correlation, and whether the registry lists
// the pair as one row.
func (f *nodeAgentSplitMonitor) assertLinked(step string, want bool) {
	f.t.Helper()
	node, agent := f.nodes[0].ID, f.host.ID
	wantAgent, wantNode := "", ""
	if want {
		wantAgent, wantNode = agent, node
	}
	snapshot := f.monitor.state.GetSnapshot()
	for _, n := range snapshot.Nodes {
		if n.ID == node && n.LinkedAgentID != wantAgent {
			f.t.Fatalf("%s: state node links agent %q, want %q", step, n.LinkedAgentID, wantAgent)
		}
	}
	for _, h := range snapshot.Hosts {
		if h.ID == agent && h.LinkedNodeID != wantNode {
			f.t.Fatalf("%s: state agent links node %q, want %q", step, h.LinkedNodeID, wantNode)
		}
	}
	if f.host.LinkedNodeID != wantNode {
		f.t.Fatalf("%s: report ingest linked node %q, want %q", step, f.host.LinkedNodeID, wantNode)
	}
	readState := f.monitor.GetUnifiedReadStateOrSnapshot()
	for _, view := range readState.Nodes() {
		if view.SourceID() == node && view.LinkedAgentID() != wantAgent {
			f.t.Fatalf("%s: read-state node links agent %q, want %q", step, view.LinkedAgentID(), wantAgent)
		}
	}
	diskHost := f.monitor.linkedHostForNode("lab", node, f.nodes[0].Name)
	if (diskHost != nil) != want || (diskHost != nil && diskHost.ID != agent) {
		f.t.Fatalf("%s: node disk source host %+v, want linked=%v", step, diskHost, want)
	}
	joined := false
	for _, resource := range f.adapter.GetAll() {
		if resource.Type == unifiedresources.ResourceTypeVM && resource.Proxmox != nil && resource.Proxmox.LinkedAgentID != wantAgent {
			f.t.Fatalf("%s: guest inherits agent %q, want %q", step, resource.Proxmox.LinkedAgentID, wantAgent)
		}
		if resource.Proxmox != nil && resource.Agent != nil {
			joined = true
		}
	}
	if joined != want {
		f.t.Fatalf("%s: registry lists the pair joined=%v, want %v", step, joined, want)
	}
	if correlated := sharedSystemAlertCorrelationForHost(f.host, snapshot.Nodes) != nil; correlated != want {
		f.t.Fatalf("%s: shared-system alert correlation=%v, want %v", step, correlated, want)
	}
}

// An operator's split of a Proxmox node and its pulse-agent (report-merge
// or unlink in the resources API) stops monitoring treating them as one
// machine: the agent's SMART inventory, --disk-exclude patterns, guest
// discovery target, deployment status and alert correlation no longer
// reach the node. An exclusion that splits neither leaves all of it alone.
func TestOperatorSplitStopsMonitoringTreatingNodeAndAgentAsLinked(t *testing.T) {
	t.Run("unrelated exclusion keeps the link", func(t *testing.T) {
		f := newNodeAgentSplitMonitor(t)
		f.cycle()
		f.cycle()
		f.assertLinked("before", true)
		joined := ""
		for _, resource := range f.adapter.GetAll() {
			if resource.Proxmox != nil && resource.Agent != nil {
				joined = resource.ID
			}
		}
		f.seedLXCFilesystems()
		if err := f.store.AddExclusion(unifiedresources.ResourceExclusion{ResourceA: joined, ResourceB: "agent-00000000deadbeef", CreatedAt: time.Now().UTC()}); err != nil {
			t.Fatal(err)
		}
		f.cycle()
		f.cycle()
		f.assertLinked("after an unrelated exclusion", true)
		if got := f.lxcFilesystemEntries(); got != 1 {
			t.Fatalf("an unrelated exclusion changed the agent's container filesystem readings to %d", got)
		}
	})

	t.Run("report-merge splits the link", func(t *testing.T) {
		f := newNodeAgentSplitMonitor(t)
		f.cycle()
		f.cycle()
		f.assertLinked("before", true)
		f.seedLXCFilesystems()
		f.recordReportMerge()
		f.cycle()
		f.cycle()
		f.assertLinked("after report-merge", false)
		if got := f.lxcFilesystemEntries(); got != 0 {
			t.Fatalf("the split left %d of the agent's container filesystem readings on the node's containers", got)
		}
	})
}

// The agents API's node link is the operator's newer decision about the
// pair, so it replaces an earlier split in the resource store, which both
// the registry and monitoring read, instead of leaving monitoring linked
// and the registry split. A link to another node lifts nothing. A split
// recorded after a manual link wins in turn and ends the manual intent, so
// the node is no longer reserved for the agent, and a later relink of the
// rows in the resources API joins them again through the link monitoring
// infers.
func TestManualNodeLinkReplacesAnOperatorSplit(t *testing.T) {
	f := newNodeAgentSplitMonitor(t)
	f.cycle()
	f.cycle()
	f.recordReportMerge()
	f.cycle()
	f.cycle()
	f.assertLinked("after report-merge", false)
	split, namingNode := f.exclusions()
	if namingNode != 1 {
		t.Fatalf("report-merge recorded %v, want one exclusion naming the node's candidate", split)
	}

	if err := f.monitor.LinkHostAgent(f.host.ID, f.nodes[1].ID); err != nil {
		t.Fatal(err)
	}
	if got, _ := f.exclusions(); !slices.Equal(got, split) {
		t.Fatalf("linking the agent to another node changed the exclusions from %v to %v", split, got)
	}
	// The node's split goes. Report-merge's exclusion of the agent's own
	// candidate from the machine-derived ID it merged under stays: it never
	// split the node (nodeAgentSplit), and it is also what splitting another
	// source off the agent records.
	if err := f.monitor.LinkHostAgent(f.host.ID, f.nodes[0].ID); err != nil {
		t.Fatal(err)
	}
	if got, namingNode := f.exclusions(); namingNode != 0 || len(got) != len(split)-1 {
		t.Fatalf("manual link left exclusions %v of %v", got, split)
	}
	f.cycle()
	f.cycle()
	f.assertLinked("after the manual link", true)

	f.recordReportMerge()
	f.cycle()
	f.cycle()
	f.assertLinked("after a split of the manual link", false)
	entry, ok := f.monitor.hostContinuityStore.Get(f.host.ID)
	if !ok || entry.NodeLinkSource != "automatic" || entry.LinkedNodeID != "" {
		t.Fatalf("split left the manual intent persisted: %+v", entry)
	}
	if reserved := f.monitor.hostContinuityStore.NodeLinkReservedByOther("another-agent", f.nodes[0].ID); reserved {
		t.Fatal("the ended manual intent still reserves the node against other agents")
	}

	var nodeRow, agentRow string
	for _, resource := range f.adapter.GetAll() {
		switch {
		case resource.Proxmox != nil && resource.Agent == nil && resource.Proxmox.SourceID == f.nodes[0].ID:
			nodeRow = resource.ID
		case resource.Agent != nil && resource.Proxmox == nil:
			agentRow = resource.ID
		}
	}
	if err := f.store.AddLink(unifiedresources.ResourceLink{ResourceA: nodeRow, ResourceB: agentRow, PrimaryID: nodeRow, CreatedAt: time.Now().UTC()}); err != nil {
		t.Fatal(err)
	}
	f.cycle()
	f.cycle()
	f.assertLinked("after a resources API relink", true)
}

// A manual node link that cannot persist its intent leaves the operator's
// split as it was: the exclusions it removed are recorded again, with their
// original times, and the pair stays apart.
func TestManualNodeLinkKeepsTheSplitWhenItsIntentCannotPersist(t *testing.T) {
	f := newNodeAgentSplitMonitor(t)
	f.cycle()
	f.cycle()
	f.recordReportMerge()
	f.cycle()
	f.cycle()
	before, err := f.store.GetExclusions()
	if err != nil {
		t.Fatal(err)
	}

	continuity := f.monitor.hostContinuityStore
	f.monitor.hostContinuityStore = nil
	if err := f.monitor.LinkHostAgent(f.host.ID, f.nodes[0].ID); err == nil {
		t.Fatal("manual link without intent storage succeeded")
	}
	f.monitor.hostContinuityStore = continuity
	after, err := f.store.GetExclusions()
	if err != nil {
		t.Fatal(err)
	}
	key := func(exclusions []unifiedresources.ResourceExclusion) []string {
		out := make([]string, 0, len(exclusions))
		for _, exclusion := range exclusions {
			out = append(out, exclusion.ResourceA+"|"+exclusion.ResourceB+"|"+exclusion.CreatedAt.UTC().Format(time.RFC3339Nano))
		}
		slices.Sort(out)
		return out
	}
	if !slices.Equal(key(before), key(after)) {
		t.Fatalf("failed manual link changed the split from %v to %v", key(before), key(after))
	}
	f.cycle()
	f.cycle()
	f.assertLinked("after the failed manual link", false)
}

// A manual node link the state holds back only because the split cannot be
// confirmed (the store is unreadable just after the link removed it) keeps
// its persisted intent, and links once the store reads again. Only a split
// the store confirms ends a manual intent.
func TestManualNodeLinkSurvivesAnUnreadableSplitStore(t *testing.T) {
	var flaky *flakyDecisionStore
	f := newNodeAgentSplitMonitorWithStore(t, func(store unifiedresources.ResourceStore) unifiedresources.ResourceStore {
		flaky = &flakyDecisionStore{ResourceStore: store}
		return flaky
	})
	f.cycle()
	f.cycle()
	f.recordReportMerge()
	f.cycle()
	f.cycle()
	f.assertLinked("after report-merge", false)
	if err := f.monitor.LinkHostAgent(f.host.ID, f.nodes[0].ID); err != nil {
		t.Fatal(err)
	}

	// A report before the adapter rebuilds: its generation still holds the
	// split the link removed, which the unreadable store cannot confirm gone,
	// so the state holds the link back while the persisted intent keeps its
	// node. (Without the persisted fallback the entry would lose the node.)
	flaky.fail.Store(true)
	held := f.reportOnly()
	if held.LinkedNodeID != "" {
		t.Fatalf("the report linked node %q although the split could not be confirmed gone", held.LinkedNodeID)
	}
	snapshot := f.monitor.state.GetSnapshot()
	for _, n := range snapshot.Nodes {
		if n.LinkedAgentID != "" {
			t.Fatalf("state links node %s to agent %q", n.ID, n.LinkedAgentID)
		}
	}
	entry, ok := f.monitor.hostContinuityStore.Get(f.host.ID)
	if !ok || entry.NodeLinkSource != "manual" || entry.LinkedNodeID != f.nodes[0].ID {
		t.Fatalf("a held-back manual link lost its persisted intent: %+v", entry)
	}

	// A rebuild that cannot read the decisions loads none, so the link
	// returns for now and the intent is unchanged.
	f.cycle()
	entry, ok = f.monitor.hostContinuityStore.Get(f.host.ID)
	if !ok || entry.NodeLinkSource != "manual" || entry.LinkedNodeID != f.nodes[0].ID {
		t.Fatalf("an unreadable split store ended the manual intent: %+v", entry)
	}
	flaky.fail.Store(false)
	f.cycle()
	f.cycle()
	f.assertLinked("after the store reads again", true)
	if entry, _ := f.monitor.hostContinuityStore.Get(f.host.ID); entry.NodeLinkSource != "manual" {
		t.Fatalf("manual intent became %q", entry.NodeLinkSource)
	}
}

// sequencedSplitDecider answers "not split" for its first calls and "split"
// from then on, as an operator's split recorded while a report is in flight
// does.
type sequencedSplitDecider struct {
	calls     atomic.Int32
	splitFrom atomic.Int32
}

func (d *sequencedSplitDecider) ProxmoxNodeAgentSplit(models.Node, models.Host) bool {
	return d.calls.Add(1) > d.splitFrom.Load()
}

// A split recorded after a report's own check but before the state stores
// the agent is applied by the state; the report then neither returns, links
// nor persists the node the state refused.
func TestReportUsesTheLinkTheStateKeptWhenASplitLandsMidReport(t *testing.T) {
	// The split is first seen by the state's own check of the agent's link
	// (call 2), or only by its check of each node linked to the agent (call 3).
	for _, seenAtCall := range []int32{1, 2} {
		t.Run(fmt.Sprintf("seen after %d checks", seenAtCall), func(t *testing.T) {
			testReportUsesTheLinkTheStateKept(t, seenAtCall)
		})
	}
}

func testReportUsesTheLinkTheStateKept(t *testing.T, splitFrom int32) {
	f := newNodeAgentSplitMonitor(t)
	f.cycle()
	f.cycle()
	f.assertLinked("before", true)
	f.seedLXCFilesystems()

	decider := &sequencedSplitDecider{}
	decider.splitFrom.Store(1 << 30)
	f.monitor.state.SetNodeAgentSplitDecider(decider)
	f.monitor.state.UpdateNodesForInstance("lab", f.nodes)
	f.monitor.state.UpsertHost(f.host)
	// The report's own check (its first call) finds no split; the state's
	// store of the agent (the next) does.
	decider.calls.Store(0)
	decider.splitFrom.Store(splitFrom)

	held := f.reportOnly()
	if held.LinkedNodeID != "" {
		t.Fatalf("the report returned node %q that the state refused", held.LinkedNodeID)
	}
	snapshot := f.monitor.state.GetSnapshot()
	for _, n := range snapshot.Nodes {
		if n.LinkedAgentID != "" {
			t.Fatalf("state links node %s to agent %q", n.ID, n.LinkedAgentID)
		}
	}
	for _, h := range snapshot.Hosts {
		if h.LinkedNodeID != "" {
			t.Fatalf("state links agent %s to node %q", h.ID, h.LinkedNodeID)
		}
	}
	entry, ok := f.monitor.hostContinuityStore.Get(f.host.ID)
	if !ok || entry.LinkedNodeID != "" {
		t.Fatalf("the report persisted node %q that the state refused: %+v", entry.LinkedNodeID, entry)
	}
	if got := f.lxcFilesystemEntries(); got != 0 {
		t.Fatalf("%d container filesystem readings from the agent outlived the split", got)
	}
}

// A confirmed split of a manual link ends the intent and clears what the
// agent cached for the node's containers even where nothing infers another
// node for the agent (inference has no node to substitute, so no later branch
// would clear it).
func TestConfirmedSplitOfAManualLinkClearsTheAgentsContainerReadings(t *testing.T) {
	f := newNodeAgentSplitMonitor(t)
	f.cycle()
	f.cycle()
	if err := f.monitor.LinkHostAgent(f.host.ID, f.nodes[0].ID); err != nil {
		t.Fatal(err)
	}
	f.cycle()
	f.cycle()
	f.assertLinked("after the manual link", true)

	// The agent no longer looks like the node to inference, and nothing but
	// the manual intent ties it to the node.
	f.report.Host.Hostname = "elsewhere"
	f.report.Host.ReportIP = ""
	f.seedLXCFilesystems()
	f.recordReportMerge()
	f.cycle()
	f.cycle()

	entry, ok := f.monitor.hostContinuityStore.Get(f.host.ID)
	if !ok || entry.NodeLinkSource != "automatic" || entry.LinkedNodeID != "" {
		t.Fatalf("a confirmed split left the manual intent: %+v", entry)
	}
	if got := f.lxcFilesystemEntries(); got != 0 {
		t.Fatalf("%d container filesystem readings from the split agent outlived its manual link", got)
	}
}

// A split agent whose name is its node's name must not hand the node its
// sensors through the unlinked-agent hostname fallback.
func TestSplitAgentSensorsDoNotReachTheNodeThroughItsName(t *testing.T) {
	f := newNodeAgentSplitMonitor(t)
	f.report.Sensors.TemperatureCelsius = map[string]float64{"cpu_package": 61}
	f.cycle()
	f.cycle()
	node := f.nodes[0]
	if temp := f.monitor.getHostAgentTemperatureForNode(node); temp == nil || temp.CPUPackage != 61 {
		t.Fatalf("a linked agent's temperature did not reach its node: %+v", temp)
	}

	f.recordReportMerge()
	f.cycle()
	f.cycle()
	f.assertLinked("after report-merge", false)
	if temp := f.monitor.getHostAgentTemperatureForNode(node); temp != nil {
		t.Fatalf("the split agent's temperature reached the node through its name: %+v", temp)
	}

	// A reading the agent supplied before the split must not outlive the
	// agent's lease because the split took it out of the node's slot.
	carried := &models.Temperature{Available: true, CPUPackage: 61, LastUpdate: f.host.LastSeen}
	if !f.monitor.carriedTemperatureOutlivesAgentLease(node, carried, time.Now().Add(time.Hour)) {
		t.Fatal("a split agent's carried reading would outlive the agent's lapsed lease")
	}
}

// Container filesystem readings an agent cached are not shown for a node the
// operator split from that agent. Seeded after the split, so no cleanup has
// run, they are ignored; a linked pair's reading is shown.
func TestAgentContainerReadingsAreNotShownForANodeSplitFromTheAgent(t *testing.T) {
	f := newNodeAgentSplitMonitor(t)
	f.cycle()
	f.cycle()
	f.recordReportMerge()
	f.cycle()
	f.cycle()
	f.assertLinked("after report-merge", false)

	// The agent's reports name no node (as when its name matches none), so
	// nothing on the report path clears what is cached.
	f.seedLXCFilesystems()
	container := models.Container{VMID: 200, Name: "ct200", Status: "running"}
	f.monitor.enrichContainerWithAgentLXCFilesystems("lab", "pve1", &container, time.Now())
	if len(container.Disks) != 0 {
		t.Fatalf("the split agent's readings were shown for the node's container: %+v", container.Disks)
	}

	// A node that is not in state under this connection (a proven duplicate
	// connection folded it into another's) is not split from the agent.
	f.monitor.proxmoxLXCFilesystemsMu.Lock()
	f.monitor.proxmoxLXCFilesystemsCache[agentLXCFilesystemCacheKey("ghost", "pve9", 300)] = agentLXCFilesystemCacheEntry{
		agentID:   f.host.ID,
		name:      "ct300",
		disks:     []models.Disk{{Mountpoint: "/", Total: 10 << 30, Used: 1 << 30}},
		expiresAt: time.Now().Add(10 * time.Minute),
	}
	f.monitor.proxmoxLXCFilesystemsMu.Unlock()
	ghost := models.Container{VMID: 300, Name: "ct300", Status: "running"}
	f.monitor.enrichContainerWithAgentLXCFilesystems("ghost", "pve9", &ghost, time.Now())
	if len(ghost.Disks) != 1 {
		t.Fatalf("a reading for a node that left this connection's slot was hidden: %+v", ghost.Disks)
	}

	// The control: with the agent linked the same reading is shown.
	linked := newNodeAgentSplitMonitor(t)
	linked.cycle()
	linked.cycle()
	linked.assertLinked("control", true)
	linked.seedLXCFilesystems()
	control := models.Container{VMID: 200, Name: "ct200", Status: "running"}
	linked.monitor.enrichContainerWithAgentLXCFilesystems("lab", "pve1", &control, time.Now())
	if len(control.Disks) != 1 {
		t.Fatalf("a linked agent's reading was not shown: %+v", control.Disks)
	}
}

// An earlier version could persist a duplicate of a tenant agent token in the
// org's own config, carrying grant state the instance record lacks. When the
// instance inventory cannot commit the revocation, the org copy must survive
// too, or the stale instance record becomes the only copy.
func TestRevokeAPITokenRestoresEveryInventoryWhenOneCannotCommit(t *testing.T) {
	if os.Geteuid() == 0 {
		t.Skip("a read-only directory does not stop root from writing")
	}
	const orgID = "t-CLIENT0003"
	record, err := config.NewAPITokenRecord("0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "agent", []string{config.ScopeAgentReport})
	if err != nil {
		t.Fatal(err)
	}
	record.OrgID = orgID
	instanceRecord := record.Clone()
	ownRecord := record.Clone()
	ownRecord.Metadata = map[string]string{"proxmox_registration_completed": "true"}

	ownDir, instanceDir := t.TempDir(), t.TempDir()
	ownPersistence := config.NewConfigPersistence(ownDir)
	instancePersistence := config.NewConfigPersistence(instanceDir)
	if err := ownPersistence.SaveAPITokens([]config.APITokenRecord{ownRecord}); err != nil {
		t.Fatal(err)
	}
	if err := instancePersistence.SaveAPITokens([]config.APITokenRecord{instanceRecord}); err != nil {
		t.Fatal(err)
	}
	if err := os.Chmod(instanceDir, 0o500); err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o700) })

	own := &config.Config{APITokens: []config.APITokenRecord{ownRecord}}
	m := &Monitor{config: own, persistence: ownPersistence}
	m.instanceTokens = APITokenInventory{
		Config:      &config.Config{APITokens: []config.APITokenRecord{instanceRecord}},
		Persistence: instancePersistence,
		OrgID:       orgID,
	}

	if _, err := m.revokeAPIToken(record.ID); err == nil {
		t.Fatal("revocation reported success although the instance inventory could not commit")
	}
	if len(own.APITokens) != 1 || own.APITokens[0].Metadata["proxmox_registration_completed"] != "true" {
		t.Fatalf("org copy after failed revocation = %+v, want the consumed grant restored", own.APITokens)
	}
	persisted, err := ownPersistence.LoadAPITokens()
	if err != nil || len(persisted) != 1 || persisted[0].Metadata["proxmox_registration_completed"] != "true" {
		t.Fatalf("persisted org copy = %+v (err %v), want the consumed grant restored", persisted, err)
	}
	if len(m.instanceTokens.Config.APITokens) != 1 {
		t.Fatal("instance record was not restored after its save failed")
	}
}