Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Record guest, node and host reboots in the change record
rcourtmanbcf4243fb947
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.mdindex 31dfe75b7f..ab3af050b9 100644--- a/docs/release-control/v6/internal/subsystems/unified-resources.md+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md@@ -6935,6 +6935,26 @@ serial/machine ID, guest key or actual set member still is. This preventsno-op `identity` configuration rows without weakening canonical matching orchanging the public resource shape. `TestResourceChangeIdentityIgnoresSetOrderAndEmptySlices`pins both the quiet and real-change cases.+Restart detection covers Proxmox nodes, VMs and LXCs and host agents as well+as Docker and Kubernetes: an uptime that goes backwards between two positive+readings is a `restart` change (`resourceRestartChanged` in+`change_emission.go`), because a reboot shorter than the offline threshold+never flips status and would otherwise leave no history. Proxmox uptime counts+only on the same node, since a move to another node starts a new process and+is recorded as a relationship change. Host-agent uptime counts only for the+same agent ID. QEMU's uptime follows its process, so an in-guest reboot or a+hard reset of a VM is seen only through a host agent inside the guest (checked+on 2026-10-29: pulse-ci reported 4.5 days of QEMU uptime 19.6 hours after a+`qmreset`). A zero reading is a stopped or unreported resource, never a reset.+The lower reading must also be the newer one by that source's+`SourceStatus.LastSeen`: snapshots are captured before publication is+serialized, so an older reading can publish after a newer one. An uptime reset+that arrives with a status change stays a `state_transition` (its+`changedFields` name the uptime too), so a host that rebooted while offline+still records the recovery that availability reporting and+`resource.status_changed` events read.+`TestBuildResourceChange_UptimeDropsThatAreNotRestarts` pins the quiet cases and+`TestBuildResourceChange_RebootWhileOfflineRecordsTheRecovery` the precedence.Change emission over registry rebuilds must diff relationships by edgeidentity only: canonical source, canonical target, type, and active state,order-insensitive (`relationshipsEquivalent` in `change_emission.go`).diff --git a/internal/unifiedresources/change_emission.go b/internal/unifiedresources/change_emission.goindex 260eb1b826..8996a74572 100644--- a/internal/unifiedresources/change_emission.go+++ b/internal/unifiedresources/change_emission.go@@ -264,6 +264,12 @@ func resourceChangedFields(before, after Resource) []string {if kubernetesRestartChanged(before, after) {changed = append(changed, "kubernetes.restarts", "kubernetes.uptimeSeconds")}+ if proxmoxRestartChanged(before, after) {+ changed = append(changed, "proxmox.uptime")+ }+ if agentRestartChanged(before, after) {+ changed = append(changed, "agent.uptimeSeconds")+ }if dockerCommandChanged(before, after) {changed = append(changed, "docker.command")}@@ -396,8 +402,48 @@ func dockerRestartChanged(before, after Resource) bool {return before.Docker.UptimeSeconds > 0 && after.Docker.UptimeSeconds > 0 && after.Docker.UptimeSeconds < before.Docker.UptimeSeconds}+// resourceRestartChanged decides the restart kind. An uptime reset that comes+// with a status change stays a state transition, so a host that rebooted while+// offline still records its recovery for availability and status events.func resourceRestartChanged(before, after Resource) bool {- return dockerRestartChanged(before, after) || kubernetesRestartChanged(before, after)+ if dockerRestartChanged(before, after) || kubernetesRestartChanged(before, after) {+ return true+ }+ return before.Status == after.Status && (proxmoxRestartChanged(before, after) || agentRestartChanged(before, after))+}++// uptimeReset reports an uptime counter that went backwards between two+// positive readings from source, the later one observed later. Snapshots are+// captured before publication is serialized, so an older reading can publish+// after a newer one; its lower uptime is not a reset. A zero reading is a+// stopped or unreported resource.+func uptimeReset(before, after Resource, source DataSource, beforeUptime, afterUptime int64) bool {+ if beforeUptime <= 0 || afterUptime <= 0 || afterUptime >= beforeUptime {+ return false+ }+ beforeSeen, afterSeen := before.SourceStatus[source].LastSeen, after.SourceStatus[source].LastSeen+ return beforeSeen.IsZero() || afterSeen.IsZero() || afterSeen.After(beforeSeen)+}++// proxmoxRestartChanged reports a node, VM or LXC whose Proxmox uptime reset+// on the same node. QEMU's uptime follows its process, so it catches a stop+// and start (including qm reboot and a host reboot) but not an in-guest reboot+// or a reset; a host agent inside the guest reports those. A move to another+// node starts a new process there and is recorded as a relationship change.+func proxmoxRestartChanged(before, after Resource) bool {+ if before.Proxmox == nil || after.Proxmox == nil || before.Proxmox.NodeName != after.Proxmox.NodeName {+ return false+ }+ return uptimeReset(before, after, SourceProxmox, before.Proxmox.Uptime, after.Proxmox.Uptime)+}++// agentRestartChanged reports an operating system reboot seen by the same host+// agent. Another agent taking over the resource is not a reboot.+func agentRestartChanged(before, after Resource) bool {+ if before.Agent == nil || after.Agent == nil || before.Agent.AgentID != after.Agent.AgentID {+ return false+ }+ return uptimeReset(before, after, SourceAgent, before.Agent.UptimeSeconds, after.Agent.UptimeSeconds)}func kubernetesRestartChanged(before, after Resource) bool {diff --git a/internal/unifiedresources/change_emission_test.go b/internal/unifiedresources/change_emission_test.goindex a3f43c7029..e569778292 100644--- a/internal/unifiedresources/change_emission_test.go+++ b/internal/unifiedresources/change_emission_test.go@@ -449,6 +449,118 @@ func TestBuildResourceChange_ClassifiesKubernetesRestartChange(t *testing.T) {}}+// A reboot shorter than the offline threshold never flips status, so the+// uptime reset is the only trace it leaves.+func TestBuildResourceChange_ClassifiesProxmoxUptimeResetAsRestart(t *testing.T) {+ before := Resource{+ ID: "vm:pve1:101",+ Type: ResourceTypeVM,+ Name: "db-01",+ Status: StatusOnline,+ Proxmox: &ProxmoxData{NodeName: "pve1", VMID: 101, Uptime: 86400},+ }+ after := before+ after.Proxmox = &ProxmoxData{NodeName: "pve1", VMID: 101, Uptime: 40}++ change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, "")+ if change == nil || change.Kind != ChangeRestart {+ t.Fatalf("change = %+v, want a restart", change)+ }+ if change.From != "online|proxmox.uptime=86400" || change.To != "online|proxmox.uptime=40" {+ t.Fatalf("From/To = %q/%q, want proxmox uptime summaries", change.From, change.To)+ }+ if !sameStringSet(mustChangedFields(t, change), []string{"proxmox.uptime"}) {+ t.Fatalf("changedFields = %+v, want proxmox.uptime", mustChangedFields(t, change))+ }+}++func TestBuildResourceChange_ClassifiesAgentUptimeResetAsRestart(t *testing.T) {+ before := Resource{+ ID: "agent:host-1",+ Type: ResourceTypeAgent,+ Name: "nas",+ Status: StatusOnline,+ Agent: &AgentData{AgentID: "agent-1", UptimeSeconds: 604800},+ }+ after := before+ after.Agent = &AgentData{AgentID: "agent-1", UptimeSeconds: 95}++ change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, "")+ if change == nil || change.Kind != ChangeRestart {+ t.Fatalf("change = %+v, want a restart", change)+ }+ if change.To != "online|agent.uptimeSeconds=95" {+ t.Fatalf("To = %q, want agent uptime summary", change.To)+ }+ if !sameStringSet(mustChangedFields(t, change), []string{"agent.uptimeSeconds"}) {+ t.Fatalf("changedFields = %+v, want agent.uptimeSeconds", mustChangedFields(t, change))+ }+}++func TestBuildResourceChange_UptimeDropsThatAreNotRestarts(t *testing.T) {+ vm := Resource{ID: "vm:pve1:101", Type: ResourceTypeVM, Status: StatusOnline}+ host := Resource{ID: "agent:host-1", Type: ResourceTypeAgent, Status: StatusOnline}+ for _, tc := range []struct {+ name string+ before, after Resource+ }{+ {"migrated to another node", withProxmox(vm, "pve1", 86400), withProxmox(vm, "pve2", 30)},+ {"stopped reading", withProxmox(vm, "pve1", 86400), withProxmox(vm, "pve1", 0)},+ {"first reading", withProxmox(vm, "pve1", 0), withProxmox(vm, "pve1", 30)},+ {"uptime grew", withProxmox(vm, "pve1", 30), withProxmox(vm, "pve1", 90)},+ {"another agent took over", withAgent(host, "agent-1", 604800), withAgent(host, "agent-2", 95)},+ {"older reading published late", seenAt(withAgent(host, "agent-1", 1030), SourceAgent, 30), seenAt(withAgent(host, "agent-1", 1000), SourceAgent, 0)},+ } {+ t.Run(tc.name, func(t *testing.T) {+ if change := buildResourceChange(tc.before, true, tc.after, true, time.Now().UTC(), nil, SourcePulseDiff, ""); change != nil && change.Kind == ChangeRestart {+ t.Fatalf("recorded a restart: %+v", change)+ }+ })+ }+}++// A host that rebooted while offline comes back with a new status and a reset+// uptime in one comparison. The recovery must stay a state transition, which+// availability reporting and status events read.+func TestBuildResourceChange_RebootWhileOfflineRecordsTheRecovery(t *testing.T) {+ before := withAgent(Resource{ID: "agent:host-1", Type: ResourceTypeAgent, Status: StatusOffline}, "agent-1", 604800)+ after := withAgent(Resource{ID: "agent:host-1", Type: ResourceTypeAgent, Status: StatusOnline}, "agent-1", 95)+ before, after = seenAt(before, SourceAgent, 0), seenAt(after, SourceAgent, 600)++ change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, "")+ if change == nil || change.Kind != ChangeStateTransition {+ t.Fatalf("change = %+v, want the recovery as a state transition", change)+ }+ if !sameStringSet(mustChangedFields(t, change), []string{"status", "agent.uptimeSeconds"}) {+ t.Fatalf("changedFields = %+v, want status and the uptime reset", mustChangedFields(t, change))+ }+}++func TestBuildResourceChange_NewerUptimeResetIsARestart(t *testing.T) {+ host := Resource{ID: "agent:host-1", Type: ResourceTypeAgent, Status: StatusOnline}+ before := seenAt(withAgent(host, "agent-1", 604800), SourceAgent, 0)+ after := seenAt(withAgent(host, "agent-1", 95), SourceAgent, 30)+ if change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, ""); change == nil || change.Kind != ChangeRestart {+ t.Fatalf("change = %+v, want a restart", change)+ }+}++func seenAt(resource Resource, source DataSource, offsetSeconds int) Resource {+ base := time.Date(2026, 10, 29, 3, 0, 0, 0, time.UTC)+ resource.SourceStatus = map[DataSource]SourceStatus{source: {LastSeen: base.Add(time.Duration(offsetSeconds) * time.Second)}}+ return resource+}++func withProxmox(resource Resource, node string, uptime int64) Resource {+ resource.Proxmox = &ProxmoxData{NodeName: node, VMID: 101, Uptime: uptime}+ return resource+}++func withAgent(resource Resource, agentID string, uptime int64) Resource {+ resource.Agent = &AgentData{AgentID: agentID, UptimeSeconds: uptime}+ return resource+}+func TestBuildResourceChange_ClassifiesIncidentAnomalyChange(t *testing.T) {before := Resource{ID: "storage:1",diff --git a/internal/unifiedresources/change_presentation.go b/internal/unifiedresources/change_presentation.goindex 6eb323e417..bf9e5d6eb8 100644--- a/internal/unifiedresources/change_presentation.go+++ b/internal/unifiedresources/change_presentation.go@@ -114,6 +114,12 @@ func resourceRestartSummary(resource Resource) string {parts = append(parts, fmt.Sprintf("kubernetes.uptimeSeconds=%d", resource.Kubernetes.UptimeSeconds))}}+ if resource.Proxmox != nil && resource.Proxmox.Uptime > 0 {+ parts = append(parts, fmt.Sprintf("proxmox.uptime=%d", resource.Proxmox.Uptime))+ }+ if resource.Agent != nil && resource.Agent.UptimeSeconds > 0 {+ parts = append(parts, fmt.Sprintf("agent.uptimeSeconds=%d", resource.Agent.UptimeSeconds))+ }return strings.Join(parts, "|")}diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.goindex 761c421916..19c33cf640 100644--- a/internal/unifiedresources/registry_test.go+++ b/internal/unifiedresources/registry_test.go@@ -197,6 +197,37 @@ func TestRegistryGenerationComparisonTreatsIdentityListsAsSets(t *testing.T) {}}+// A guest that rebooted between two polls stays "running", so its Proxmox+// uptime reset is the only trace. Ingest stamps the poll time per source, and+// an older poll published late must not read as a reboot.+func TestRegistryGenerationComparisonRecordsProxmoxUptimeResetOnce(t *testing.T) {+ store := NewMemoryStore()+ polled := time.Date(2026, 10, 29, 3, 0, 0, 0, time.UTC)+ vm := models.VM{ID: "pve:node:105", Instance: "pve", Node: "node", VMID: 105, Type: "qemu", Status: "running", Name: "guest", Uptime: 86400, LastSeen: polled}+ generation := func(uptime int64, seen time.Time) *ResourceRegistry {+ rr := NewRegistry(store)+ vm.Uptime, vm.LastSeen = uptime, seen+ rr.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})+ return rr+ }++ before := generation(86400, polled)+ rebooted := generation(40, polled.Add(time.Minute))+ recordRegistryChangesBetweenGenerations(before, rebooted, polled.Add(time.Minute), nil, SourcePulseDiff, "")+ if len(store.changes) != 1 || store.changes[0].Kind != ChangeRestart {+ t.Fatalf("reboot between polls: %+v", store.changes)+ }++ newer := generation(100, polled.Add(2*time.Minute))+ stale := generation(70, polled.Add(90*time.Second))+ recordRegistryChangesBetweenGenerations(newer, stale, polled.Add(2*time.Minute), nil, SourcePulseDiff, "")+ for _, change := range store.changes[1:] {+ if change.Kind == ChangeRestart {+ t.Fatalf("older poll published late recorded a restart: %+v", store.changes)+ }+ }+}+// TestMemoryStore_RecordActionAuditAppliesRedaction is an integration check// at the registry-store boundary. The MemoryStore is the backing store the// registry uses in tests and contract examples, and operator-authored audit