Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Record guest, node and host reboots in the change record

rcourtmanbcf4243fb947
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md
index 31dfe75b7f..ab3af050b9 100644
--- a/docs/release-control/v6/internal/subsystems/unified-resources.md
+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md
@@ -6935,6 +6935,26 @@ serial/machine ID, guest key or actual set member still is. This prevents
no-op `identity` configuration rows without weakening canonical matching or
changing the public resource shape. `TestResourceChangeIdentityIgnoresSetOrderAndEmptySlices`
pins both the quiet and real-change cases.
+Restart detection covers Proxmox nodes, VMs and LXCs and host agents as well
+as Docker and Kubernetes: an uptime that goes backwards between two positive
+readings is a `restart` change (`resourceRestartChanged` in
+`change_emission.go`), because a reboot shorter than the offline threshold
+never flips status and would otherwise leave no history. Proxmox uptime counts
+only on the same node, since a move to another node starts a new process and
+is recorded as a relationship change. Host-agent uptime counts only for the
+same agent ID. QEMU's uptime follows its process, so an in-guest reboot or a
+hard reset of a VM is seen only through a host agent inside the guest (checked
+on 2026-10-29: pulse-ci reported 4.5 days of QEMU uptime 19.6 hours after a
+`qmreset`). A zero reading is a stopped or unreported resource, never a reset.
+The lower reading must also be the newer one by that source's
+`SourceStatus.LastSeen`: snapshots are captured before publication is
+serialized, so an older reading can publish after a newer one. An uptime reset
+that arrives with a status change stays a `state_transition` (its
+`changedFields` name the uptime too), so a host that rebooted while offline
+still records the recovery that availability reporting and
+`resource.status_changed` events read.
+`TestBuildResourceChange_UptimeDropsThatAreNotRestarts` pins the quiet cases and
+`TestBuildResourceChange_RebootWhileOfflineRecordsTheRecovery` the precedence.
Change emission over registry rebuilds must diff relationships by edge
identity only: canonical source, canonical target, type, and active state,
order-insensitive (`relationshipsEquivalent` in `change_emission.go`).
diff --git a/internal/unifiedresources/change_emission.go b/internal/unifiedresources/change_emission.go
index 260eb1b826..8996a74572 100644
--- a/internal/unifiedresources/change_emission.go
+++ b/internal/unifiedresources/change_emission.go
@@ -264,6 +264,12 @@ func resourceChangedFields(before, after Resource) []string {
if kubernetesRestartChanged(before, after) {
changed = append(changed, "kubernetes.restarts", "kubernetes.uptimeSeconds")
}
+ if proxmoxRestartChanged(before, after) {
+ changed = append(changed, "proxmox.uptime")
+ }
+ if agentRestartChanged(before, after) {
+ changed = append(changed, "agent.uptimeSeconds")
+ }
if dockerCommandChanged(before, after) {
changed = append(changed, "docker.command")
}
@@ -396,8 +402,48 @@ func dockerRestartChanged(before, after Resource) bool {
return before.Docker.UptimeSeconds > 0 && after.Docker.UptimeSeconds > 0 && after.Docker.UptimeSeconds < before.Docker.UptimeSeconds
}
+// resourceRestartChanged decides the restart kind. An uptime reset that comes
+// with a status change stays a state transition, so a host that rebooted while
+// offline still records its recovery for availability and status events.
func resourceRestartChanged(before, after Resource) bool {
- return dockerRestartChanged(before, after) || kubernetesRestartChanged(before, after)
+ if dockerRestartChanged(before, after) || kubernetesRestartChanged(before, after) {
+ return true
+ }
+ return before.Status == after.Status && (proxmoxRestartChanged(before, after) || agentRestartChanged(before, after))
+}
+
+// uptimeReset reports an uptime counter that went backwards between two
+// positive readings from source, the later one observed later. Snapshots are
+// captured before publication is serialized, so an older reading can publish
+// after a newer one; its lower uptime is not a reset. A zero reading is a
+// stopped or unreported resource.
+func uptimeReset(before, after Resource, source DataSource, beforeUptime, afterUptime int64) bool {
+ if beforeUptime <= 0 || afterUptime <= 0 || afterUptime >= beforeUptime {
+ return false
+ }
+ beforeSeen, afterSeen := before.SourceStatus[source].LastSeen, after.SourceStatus[source].LastSeen
+ return beforeSeen.IsZero() || afterSeen.IsZero() || afterSeen.After(beforeSeen)
+}
+
+// proxmoxRestartChanged reports a node, VM or LXC whose Proxmox uptime reset
+// on the same node. QEMU's uptime follows its process, so it catches a stop
+// and start (including qm reboot and a host reboot) but not an in-guest reboot
+// or a reset; a host agent inside the guest reports those. A move to another
+// node starts a new process there and is recorded as a relationship change.
+func proxmoxRestartChanged(before, after Resource) bool {
+ if before.Proxmox == nil || after.Proxmox == nil || before.Proxmox.NodeName != after.Proxmox.NodeName {
+ return false
+ }
+ return uptimeReset(before, after, SourceProxmox, before.Proxmox.Uptime, after.Proxmox.Uptime)
+}
+
+// agentRestartChanged reports an operating system reboot seen by the same host
+// agent. Another agent taking over the resource is not a reboot.
+func agentRestartChanged(before, after Resource) bool {
+ if before.Agent == nil || after.Agent == nil || before.Agent.AgentID != after.Agent.AgentID {
+ return false
+ }
+ return uptimeReset(before, after, SourceAgent, before.Agent.UptimeSeconds, after.Agent.UptimeSeconds)
}
func kubernetesRestartChanged(before, after Resource) bool {
diff --git a/internal/unifiedresources/change_emission_test.go b/internal/unifiedresources/change_emission_test.go
index a3f43c7029..e569778292 100644
--- a/internal/unifiedresources/change_emission_test.go
+++ b/internal/unifiedresources/change_emission_test.go
@@ -449,6 +449,118 @@ func TestBuildResourceChange_ClassifiesKubernetesRestartChange(t *testing.T) {
}
}
+// A reboot shorter than the offline threshold never flips status, so the
+// uptime reset is the only trace it leaves.
+func TestBuildResourceChange_ClassifiesProxmoxUptimeResetAsRestart(t *testing.T) {
+ before := Resource{
+ ID: "vm:pve1:101",
+ Type: ResourceTypeVM,
+ Name: "db-01",
+ Status: StatusOnline,
+ Proxmox: &ProxmoxData{NodeName: "pve1", VMID: 101, Uptime: 86400},
+ }
+ after := before
+ after.Proxmox = &ProxmoxData{NodeName: "pve1", VMID: 101, Uptime: 40}
+
+ change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, "")
+ if change == nil || change.Kind != ChangeRestart {
+ t.Fatalf("change = %+v, want a restart", change)
+ }
+ if change.From != "online|proxmox.uptime=86400" || change.To != "online|proxmox.uptime=40" {
+ t.Fatalf("From/To = %q/%q, want proxmox uptime summaries", change.From, change.To)
+ }
+ if !sameStringSet(mustChangedFields(t, change), []string{"proxmox.uptime"}) {
+ t.Fatalf("changedFields = %+v, want proxmox.uptime", mustChangedFields(t, change))
+ }
+}
+
+func TestBuildResourceChange_ClassifiesAgentUptimeResetAsRestart(t *testing.T) {
+ before := Resource{
+ ID: "agent:host-1",
+ Type: ResourceTypeAgent,
+ Name: "nas",
+ Status: StatusOnline,
+ Agent: &AgentData{AgentID: "agent-1", UptimeSeconds: 604800},
+ }
+ after := before
+ after.Agent = &AgentData{AgentID: "agent-1", UptimeSeconds: 95}
+
+ change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, "")
+ if change == nil || change.Kind != ChangeRestart {
+ t.Fatalf("change = %+v, want a restart", change)
+ }
+ if change.To != "online|agent.uptimeSeconds=95" {
+ t.Fatalf("To = %q, want agent uptime summary", change.To)
+ }
+ if !sameStringSet(mustChangedFields(t, change), []string{"agent.uptimeSeconds"}) {
+ t.Fatalf("changedFields = %+v, want agent.uptimeSeconds", mustChangedFields(t, change))
+ }
+}
+
+func TestBuildResourceChange_UptimeDropsThatAreNotRestarts(t *testing.T) {
+ vm := Resource{ID: "vm:pve1:101", Type: ResourceTypeVM, Status: StatusOnline}
+ host := Resource{ID: "agent:host-1", Type: ResourceTypeAgent, Status: StatusOnline}
+ for _, tc := range []struct {
+ name string
+ before, after Resource
+ }{
+ {"migrated to another node", withProxmox(vm, "pve1", 86400), withProxmox(vm, "pve2", 30)},
+ {"stopped reading", withProxmox(vm, "pve1", 86400), withProxmox(vm, "pve1", 0)},
+ {"first reading", withProxmox(vm, "pve1", 0), withProxmox(vm, "pve1", 30)},
+ {"uptime grew", withProxmox(vm, "pve1", 30), withProxmox(vm, "pve1", 90)},
+ {"another agent took over", withAgent(host, "agent-1", 604800), withAgent(host, "agent-2", 95)},
+ {"older reading published late", seenAt(withAgent(host, "agent-1", 1030), SourceAgent, 30), seenAt(withAgent(host, "agent-1", 1000), SourceAgent, 0)},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ if change := buildResourceChange(tc.before, true, tc.after, true, time.Now().UTC(), nil, SourcePulseDiff, ""); change != nil && change.Kind == ChangeRestart {
+ t.Fatalf("recorded a restart: %+v", change)
+ }
+ })
+ }
+}
+
+// A host that rebooted while offline comes back with a new status and a reset
+// uptime in one comparison. The recovery must stay a state transition, which
+// availability reporting and status events read.
+func TestBuildResourceChange_RebootWhileOfflineRecordsTheRecovery(t *testing.T) {
+ before := withAgent(Resource{ID: "agent:host-1", Type: ResourceTypeAgent, Status: StatusOffline}, "agent-1", 604800)
+ after := withAgent(Resource{ID: "agent:host-1", Type: ResourceTypeAgent, Status: StatusOnline}, "agent-1", 95)
+ before, after = seenAt(before, SourceAgent, 0), seenAt(after, SourceAgent, 600)
+
+ change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, "")
+ if change == nil || change.Kind != ChangeStateTransition {
+ t.Fatalf("change = %+v, want the recovery as a state transition", change)
+ }
+ if !sameStringSet(mustChangedFields(t, change), []string{"status", "agent.uptimeSeconds"}) {
+ t.Fatalf("changedFields = %+v, want status and the uptime reset", mustChangedFields(t, change))
+ }
+}
+
+func TestBuildResourceChange_NewerUptimeResetIsARestart(t *testing.T) {
+ host := Resource{ID: "agent:host-1", Type: ResourceTypeAgent, Status: StatusOnline}
+ before := seenAt(withAgent(host, "agent-1", 604800), SourceAgent, 0)
+ after := seenAt(withAgent(host, "agent-1", 95), SourceAgent, 30)
+ if change := buildResourceChange(before, true, after, true, time.Now().UTC(), nil, SourcePulseDiff, ""); change == nil || change.Kind != ChangeRestart {
+ t.Fatalf("change = %+v, want a restart", change)
+ }
+}
+
+func seenAt(resource Resource, source DataSource, offsetSeconds int) Resource {
+ base := time.Date(2026, 10, 29, 3, 0, 0, 0, time.UTC)
+ resource.SourceStatus = map[DataSource]SourceStatus{source: {LastSeen: base.Add(time.Duration(offsetSeconds) * time.Second)}}
+ return resource
+}
+
+func withProxmox(resource Resource, node string, uptime int64) Resource {
+ resource.Proxmox = &ProxmoxData{NodeName: node, VMID: 101, Uptime: uptime}
+ return resource
+}
+
+func withAgent(resource Resource, agentID string, uptime int64) Resource {
+ resource.Agent = &AgentData{AgentID: agentID, UptimeSeconds: uptime}
+ return resource
+}
+
func TestBuildResourceChange_ClassifiesIncidentAnomalyChange(t *testing.T) {
before := Resource{
ID: "storage:1",
diff --git a/internal/unifiedresources/change_presentation.go b/internal/unifiedresources/change_presentation.go
index 6eb323e417..bf9e5d6eb8 100644
--- a/internal/unifiedresources/change_presentation.go
+++ b/internal/unifiedresources/change_presentation.go
@@ -114,6 +114,12 @@ func resourceRestartSummary(resource Resource) string {
parts = append(parts, fmt.Sprintf("kubernetes.uptimeSeconds=%d", resource.Kubernetes.UptimeSeconds))
}
}
+ if resource.Proxmox != nil && resource.Proxmox.Uptime > 0 {
+ parts = append(parts, fmt.Sprintf("proxmox.uptime=%d", resource.Proxmox.Uptime))
+ }
+ if resource.Agent != nil && resource.Agent.UptimeSeconds > 0 {
+ parts = append(parts, fmt.Sprintf("agent.uptimeSeconds=%d", resource.Agent.UptimeSeconds))
+ }
return strings.Join(parts, "|")
}
diff --git a/internal/unifiedresources/registry_test.go b/internal/unifiedresources/registry_test.go
index 761c421916..19c33cf640 100644
--- a/internal/unifiedresources/registry_test.go
+++ b/internal/unifiedresources/registry_test.go
@@ -197,6 +197,37 @@ func TestRegistryGenerationComparisonTreatsIdentityListsAsSets(t *testing.T) {
}
}
+// A guest that rebooted between two polls stays "running", so its Proxmox
+// uptime reset is the only trace. Ingest stamps the poll time per source, and
+// an older poll published late must not read as a reboot.
+func TestRegistryGenerationComparisonRecordsProxmoxUptimeResetOnce(t *testing.T) {
+ store := NewMemoryStore()
+ polled := time.Date(2026, 10, 29, 3, 0, 0, 0, time.UTC)
+ vm := models.VM{ID: "pve:node:105", Instance: "pve", Node: "node", VMID: 105, Type: "qemu", Status: "running", Name: "guest", Uptime: 86400, LastSeen: polled}
+ generation := func(uptime int64, seen time.Time) *ResourceRegistry {
+ rr := NewRegistry(store)
+ vm.Uptime, vm.LastSeen = uptime, seen
+ rr.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})
+ return rr
+ }
+
+ before := generation(86400, polled)
+ rebooted := generation(40, polled.Add(time.Minute))
+ recordRegistryChangesBetweenGenerations(before, rebooted, polled.Add(time.Minute), nil, SourcePulseDiff, "")
+ if len(store.changes) != 1 || store.changes[0].Kind != ChangeRestart {
+ t.Fatalf("reboot between polls: %+v", store.changes)
+ }
+
+ newer := generation(100, polled.Add(2*time.Minute))
+ stale := generation(70, polled.Add(90*time.Second))
+ recordRegistryChangesBetweenGenerations(newer, stale, polled.Add(2*time.Minute), nil, SourcePulseDiff, "")
+ for _, change := range store.changes[1:] {
+ if change.Kind == ChangeRestart {
+ t.Fatalf("older poll published late recorded a restart: %+v", store.changes)
+ }
+ }
+}
+
// TestMemoryStore_RecordActionAuditAppliesRedaction is an integration check
// at the registry-store boundary. The MemoryStore is the backing store the
// registry uses in tests and contract examples, and operator-authored audit