Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Preserve correlated guest disks without the unrelated SMART change
pulse-triage[bot]66b828c93884
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex e3f834ffee..09c9e8100b 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -5637,3 +5637,22 @@ client through four normal fourteen-job/five-node polls into model/wire state,with zero-count error, denied status, recovery and another-instance isolation.The unchanged replication table consumes those explicit outcomes. These localHTTP controls are not native restart recovery; issue #2606 remains separate.++### Next guest polls preserve correlated agent filesystem inventory++The previous-guest context carries disks and their original agent source time+from a manually correlated VM/container even when the standalone host row was+removed, and even if agent memory is unavailable. Standalone linked hosts use+the same source-owned eligibility; a current platform row cannot renew agent+disk evidence. The existing VM disk preference consumes this inventory without+issuing another guest-agent read. Empty replacements, expiry and withdrawal+remove eligibility; a new ordinary report restores it. Guest-read safety and+backup deferrals remain independent of usable Pulse-agent filesystem data.++`guest_linked_agent_disks_test.go` carries actual registry correlation, retained+links and automatic link hints through ordinary VM construction and canonical+projection. It covers disk-only evidence, source time, missing links, identical+VMIDs in separate instances, expiry/offline/empty/withdrawal/recovery and a+backup lock with zero filesystem calls. Sequential snapshot replacement pins+inventory removal/recovery without restarting the monitor. These are synthetic+source controls, not a native cause, thaw or installed recovery for #2616/#2619.diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.mdindex 7d81104818..49543500bb 100644--- a/docs/release-control/v6/internal/subsystems/unified-resources.md+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md@@ -6299,3 +6299,17 @@ above the hosts table without its filter toolbar. Nothing renders whenvCenter reports no signals. The Overview's own model, workload snapshot andnavigation facets still come from the Overview query; the Health tab keepsthe filterable table.++### Correlated guest disks retain the independent agent source++`VMView.LinkedAgentDisks`, `ContainerView.LinkedAgentDisks` and+`HostView.CurrentAgentDisks` expose detached agent-owned filesystem inventory+with the agent source's own receipt time, independently of agent memory.+Missing, non-online, expired, stale, undated or future agent sources cannot+supply current disks just because the platform row keeps reporting. The+registry's configured source lease remains authoritative; this adds no new+expiry duration. Existing `HostView.Disks` retains last-known presentation.++`linked_agent_disks_test.go` covers selection, source timestamps, nil views+and detached inventory across all three views. No public model field, inferred+link, guest command or safety-pause clearance is introduced.diff --git a/internal/monitoring/guest_linked_agent_disks_test.go b/internal/monitoring/guest_linked_agent_disks_test.gonew file mode 100644index 0000000000..0779195938--- /dev/null+++ b/internal/monitoring/guest_linked_agent_disks_test.go@@ -0,0 +1,176 @@+package monitoring++import (+ "context"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+ "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"+)++// Ordinary guest construction uses existing agent evidence, never a new QGA+// probe. A backup lock remains a deferral even when independent disks exist.+type linkedDiskPollClient struct {+ stubPVEClient+ fsCalls int+ locked bool+}++func (c *linkedDiskPollClient) GetVMStatus(context.Context, string, int) (*proxmox.VMStatus, error) {+ status := &proxmox.VMStatus{MaxMem: 8000, Mem: 2000}+ if c.locked {+ status.Lock = "backup"+ status.Agent = proxmox.VMAgentField{Value: 1}+ }+ return status, nil+}+func (c *linkedDiskPollClient) GetVMFSInfo(context.Context, string, int) ([]proxmox.VMFileSystem, error) {+ c.fsCalls+++ return nil, nil+}++func TestLinkedGuestDisksNormalPollLifecycle(t *testing.T) {+ for _, merged := range []bool{false, true} {+ name := "automatic-hint"+ if merged {+ name = "retained-manual-link"+ }+ t.Run(name, func(t *testing.T) {+ seen := time.Now().Add(-5 * time.Second)+ guestID := makeGuestID("pve-a", "node1", 111)+ otherID := makeGuestID("pve-b", "node1", 111)+ store := unifiedresources.NewMemoryStore()+ if merged {+ if err := store.AddLink(unifiedresources.ResourceLink{ResourceA: "vm-test", ResourceB: "agent-test", PrimaryID: "agent-test"}); err != nil {+ t.Fatal(err)+ }+ }+ for _, phase := range []struct {+ name string+ status string+ seen time.Time+ diskUsed int64+ empty, withdrawn, unlink bool+ want bool+ }{+ {name: "current", status: "online", seen: seen, diskUsed: 620, want: true},+ {name: "expired-despite-live-platform", status: "online", seen: seen.Add(-time.Hour), diskUsed: 620},+ {name: "offline", status: "offline", seen: seen, diskUsed: 620},+ {name: "empty-replacement", status: "online", seen: seen, empty: true},+ {name: "withdrawal", withdrawn: true},+ {name: "missing-link", status: "online", seen: seen, diskUsed: 620, unlink: true},+ {name: "recovery-after-registry-restart", status: "online", seen: seen, diskUsed: 300, want: true},+ } {+ t.Run(phase.name, func(t *testing.T) {+ // Reconstruct from sources plus the durable link on each phase. No old+ // matcher, remembered fallback inventory or revived source is required.+ phaseStore := store+ if phase.unlink {+ phaseStore = unifiedresources.NewMemoryStore()+ }+ registry := unifiedresources.NewRegistry(phaseStore)+ resources := []unifiedresources.Resource{+ {ID: "vm-test", Type: unifiedresources.ResourceTypeVM, Name: "windows", Status: unifiedresources.StatusOnline, LastSeen: time.Now(), Sources: []unifiedresources.DataSource{unifiedresources.SourceProxmox}, Proxmox: &unifiedresources.ProxmoxData{SourceID: guestID, Instance: "pve-a", NodeName: "node1", VMID: 111, RuntimeStatus: "running"}},+ {ID: "other-vm", Type: unifiedresources.ResourceTypeVM, Name: "other-windows", Status: unifiedresources.StatusOnline, LastSeen: time.Now(), Sources: []unifiedresources.DataSource{unifiedresources.SourceProxmox}, Proxmox: &unifiedresources.ProxmoxData{SourceID: otherID, Instance: "pve-b", NodeName: "node1", VMID: 111, RuntimeStatus: "running"}},+ }+ if !phase.withdrawn {+ disks := []unifiedresources.DiskInfo{{Total: 1000, Used: phase.diskUsed, Free: 1000 - phase.diskUsed, Usage: float64(phase.diskUsed) / 10, Mountpoint: `C:\`, Device: "C:", Filesystem: "ntfs"}}+ if phase.empty {+ disks = nil+ }+ link := guestID+ if phase.unlink {+ link = ""+ }+ resources = append(resources, unifiedresources.Resource{ID: "agent-test", Type: unifiedresources.ResourceTypeAgent, Name: "windows-agent", Status: unifiedresources.ResourceStatus(phase.status), LastSeen: phase.seen, Sources: []unifiedresources.DataSource{unifiedresources.SourceAgent}, Agent: &unifiedresources.AgentData{AgentID: "agent-test", LinkedVMID: link, Stale: phase.status == "offline", Disks: disks}})+ }+ registry.IngestResources(resources)+ mon := &Monitor{state: models.NewState(), rateTracker: NewRateTracker(), config: &config.Config{}, resourceStore: unifiedresources.NewMonitorAdapter(registry)}+ prev := mon.previousGuestContextForInstance("pve-a")+ agent := prev.hostAgentsByVMID[guestID]+ if phase.want && (!agent.LastSeen.Equal(phase.seen) || len(agent.Disks) != 1) {+ t.Fatalf("agent evidence dropped or renewed by platform: %+v", agent)+ }+ for _, locked := range []bool{false, true} {+ client := &linkedDiskPollClient{locked: locked}+ vm, _, _, _, _, ok := mon.buildVMFromClusterResource(context.Background(), "pve-a", proxmox.ClusterResource{Type: "qemu", Status: "running", Node: "node1", VMID: 111, Name: "windows", MaxDisk: 2000, MaxMem: 8000, Mem: 2000}, client, guestID, prev.hostAgentsByVMID, nil)+ if !ok {+ t.Fatal("guest omitted")+ }+ if phase.want {+ if vm.Disk.Total != 1000 || vm.Disk.Used != phase.diskUsed || vm.Disk.Usage != float64(phase.diskUsed)/10 || len(vm.Disks) != 1 || vm.DiskStatusReason != "" {+ t.Fatalf("ordinary poll lost current agent disks: %+v", vm)+ }+ } else if vm.Disk.Usage != -1 || len(vm.Disks) != 0 || vm.DiskStatusReason == "" {+ t.Fatalf("unavailable agent invented disk reading: %+v", vm)+ }+ if client.fsCalls != 0 {+ t.Fatal("fallback issued QGA probe")+ }+ if locked && vm.GuestAgentStatus != "deferred" {+ t.Fatal("independent disks cleared guest-read safety deferral")+ }+ projection := unifiedresources.NewRegistry(nil)+ projection.IngestSnapshot(models.StateSnapshot{VMs: []models.VM{vm}})+ view := projection.VMs()[0]+ if view.DiskStatusReason() != vm.DiskStatusReason {+ t.Fatal("canonical view disagrees with poll")+ }+ resource, _ := projection.Get(view.ID())+ if phase.want {+ if view.DiskUsed() != vm.Disk.Used || view.DiskPercent() != vm.Disk.Usage {+ t.Fatal("canonical current disk disagrees with poll")+ }+ } else if resource.Metrics != nil && resource.Metrics.Disk != nil {+ t.Fatal("canonical projection invented an unavailable disk metric")+ }+ }+ other := mon.previousGuestContextForInstance("pve-b")+ if _, _, ok := resolveGuestDiskFromLinkedHostAgent(otherID, other.hostAgentsByVMID); ok {+ t.Fatal("same VMID across instances inherited another guest's disk")+ }+ })+ }+ })+ }+}++func TestLinkedGuestDiskInventoryReplacement(t *testing.T) {+ id := makeGuestID("pve-a", "node1", 111)+ registry := unifiedresources.NewRegistry(nil)+ adapter := unifiedresources.NewMonitorAdapter(registry)+ mon := &Monitor{state: models.NewState(), resourceStore: adapter}+ snapshot := models.StateSnapshot{VMs: []models.VM{{ID: id, Instance: "pve-a", Node: "node1", VMID: 111, Name: "windows", Status: "running", LastSeen: time.Now()}}}+ for _, phase := range []struct {+ name string+ disks []models.Disk+ withdrawn bool+ want bool+ }{+ {name: "current", disks: []models.Disk{{Total: 1000, Used: 620, Free: 380, Usage: 62, Mountpoint: `C:\`, Type: "ntfs"}}, want: true},+ {name: "empty-replacement"},+ {name: "recovery", disks: []models.Disk{{Total: 1000, Used: 300, Free: 700, Usage: 30, Mountpoint: `C:\`, Type: "ntfs"}}, want: true},+ {name: "withdrawal", withdrawn: true},+ } {+ t.Run(phase.name, func(t *testing.T) {+ snapshot.Hosts = []models.Host{{ID: "agent", Hostname: "windows-agent", Status: "online", LastSeen: time.Now(), LinkedVMID: id, Disks: phase.disks}}+ if phase.withdrawn {+ snapshot.Hosts = nil+ }+ // The monitor's authoritative snapshot refresh rebuilds its registry;+ // raw registry ingestion is deliberately cumulative, not deletion.+ adapter.PopulateFromSnapshot(snapshot)+ prev := mon.previousGuestContextForInstance("pve-a")+ summary, _, ok := resolveGuestDiskFromLinkedHostAgent(id, prev.hostAgentsByVMID)+ if ok != phase.want {+ t.Fatalf("replacement eligibility=%v want=%v", ok, phase.want)+ }+ if ok && summary.Used != phase.disks[0].Used {+ t.Fatalf("previous inventory survived replacement: %+v", summary)+ }+ })+ }+}diff --git a/internal/monitoring/monitor_previous_state.go b/internal/monitoring/monitor_previous_state.goindex 79568436ef..f714447abb 100644--- a/internal/monitoring/monitor_previous_state.go+++ b/internal/monitoring/monitor_previous_state.go@@ -44,8 +44,13 @@ func (m *Monitor) previousGuestContextForInstance(instanceName string) previousGguestID := makeGuestID(modelVM.Instance, modelVM.Node, modelVM.VMID)if guestID != "" {ctx.vmsByID[guestID] = modelVM- if memory, ok := vm.LinkedAgentMemory(); ok {- ctx.hostAgentsByVMID[guestID] = models.Host{LinkedVMID: guestID, Status: "online", Memory: memory}+ agent := models.Host{LinkedVMID: guestID, Status: "online"}+ memory, hasMemory := vm.LinkedAgentMemory()+ agent.Memory = memory+ disks, observedAt, hasDisks := vm.LinkedAgentDisks()+ agent.Disks, agent.LastSeen = guestDisksFromReadStateView(disks), observedAt+ if hasMemory || hasDisks {+ ctx.hostAgentsByVMID[guestID] = agent}}}@@ -62,8 +67,13 @@ func (m *Monitor) previousGuestContextForInstance(instanceName string) previousGguestID := makeGuestID(container.Instance, container.Node, container.VMID)if guestID != "" {ctx.containersByID[guestID] = container- if memory, ok := ct.LinkedAgentMemory(); ok {- ctx.hostAgentsByVMID[guestID] = models.Host{LinkedVMID: guestID, Status: "online", Memory: memory}+ agent := models.Host{LinkedContainerID: guestID, Status: "online"}+ memory, hasMemory := ct.LinkedAgentMemory()+ agent.Memory = memory+ disks, observedAt, hasDisks := ct.LinkedAgentDisks()+ agent.Disks, agent.LastSeen = guestDisksFromReadStateView(disks), observedAt+ if hasMemory || hasDisks {+ ctx.hostAgentsByVMID[guestID] = agent}}if container.VMID > 0 && (strings.EqualFold(strings.TrimSpace(container.Type), "oci") || container.IsOCI) {@@ -187,14 +197,19 @@ func previousHostFromView(host *unifiedresources.HostView) models.Host {if host == nil {return models.Host{}}+ disks, _, _ := host.CurrentAgentDisks()+ observedAt := host.LastSeen()+ if status, ok := host.SourceStatus(unifiedresources.SourceAgent); ok {+ observedAt = status.LastSeen+ }return models.Host{ID: host.ID(),Hostname: host.Hostname(),Status: string(host.Status()),LinkedVMID: host.LinkedVMID(),LinkedContainerID: host.LinkedContainerID(),- LastSeen: host.LastSeen(),- Disks: guestDisksFromReadStateView(host.Disks()),+ LastSeen: observedAt,+ Disks: guestDisksFromReadStateView(disks),Memory: models.Memory{Used: host.MemoryUsed(),Total: host.MemoryTotal(),diff --git a/internal/unifiedresources/linked_agent_disks_test.go b/internal/unifiedresources/linked_agent_disks_test.gonew file mode 100644index 0000000000..0964c0b3f1--- /dev/null+++ b/internal/unifiedresources/linked_agent_disks_test.go@@ -0,0 +1,65 @@+package unifiedresources++import (+ "reflect"+ "testing"+ "time"+)++func TestLinkedAgentDiskInventorySourceBoundaries(t *testing.T) {+ now := time.Now()+ seen := now.Add(-5 * time.Second)+ for _, tc := range []struct {+ name, status string+ stale, missing, empty bool+ seen time.Time+ want bool+ }{+ {name: "current", status: "online", seen: seen, want: true},+ {name: "offline", status: "offline", seen: seen},+ {name: "expired", status: "stale", seen: seen},+ {name: "agent-stale", status: "online", seen: seen, stale: true},+ {name: "missing-source", seen: seen, missing: true},+ {name: "undated", status: "online"},+ {name: "future", status: "online", seen: now.Add(time.Hour)},+ {name: "empty-inventory", status: "online", seen: seen, empty: true},+ } {+ t.Run(tc.name, func(t *testing.T) {+ r := &Resource{LastSeen: now, Agent: &AgentData{Stale: tc.stale, Disks: []DiskInfo{{Total: 1000, Used: 620, Free: 380, Usage: 62, Mountpoint: `C:\`, Device: "C:", Filesystem: "ntfs"}}}, SourceStatus: map[DataSource]SourceStatus{SourceAgent: {Status: tc.status, LastSeen: tc.seen}}}+ if tc.missing {+ delete(r.SourceStatus, SourceAgent)+ }+ if tc.empty {+ r.Agent.Disks = nil+ }+ for _, read := range []func() ([]DiskInfo, time.Time, bool){NewVMView(r).LinkedAgentDisks, NewContainerView(r).LinkedAgentDisks, NewHostView(r).CurrentAgentDisks} {+ disks, stamp, ok := read()+ if ok != tc.want {+ t.Fatalf("inventory eligibility=%v want %v", ok, tc.want)+ }+ if !ok {+ if disks != nil || !stamp.IsZero() {+ t.Fatal("unavailable source leaked current inventory")+ }+ continue+ }+ if !stamp.Equal(seen) || !reflect.DeepEqual(disks, r.Agent.Disks) {+ t.Fatalf("lost source identity/time: %v %+v", stamp, disks)+ }+ disks[0].Used = 999+ if r.Agent.Disks[0].Used != 620 {+ t.Fatal("view aliases owned inventory")+ }+ }+ // Last-known presentation remains available even when selection is denied.+ if !reflect.DeepEqual(NewHostView(r).Disks(), r.Agent.Disks) {+ t.Fatal("selection erased retained inventory")+ }+ })+ }+ for _, read := range []func() ([]DiskInfo, time.Time, bool){(VMView{}).LinkedAgentDisks, (ContainerView{}).LinkedAgentDisks, (HostView{}).CurrentAgentDisks} {+ if disks, stamp, ok := read(); ok || disks != nil || !stamp.IsZero() {+ t.Fatal("nil view invents inventory")+ }+ }+}diff --git a/internal/unifiedresources/views.go b/internal/unifiedresources/views.goindex 81667e27e1..ac2fe90a23 100644--- a/internal/unifiedresources/views.go+++ b/internal/unifiedresources/views.go@@ -81,6 +81,25 @@ func (v VMView) LinkedAgentMemory() (models.Memory, bool) {return linkedAgentMemoryFromResource(v.r)}+// LinkedAgentDisks reads the agent-owned inventory after host/guest correlation.+// The platform row's LastSeen must never renew the agent's reporting lease.+func (v VMView) LinkedAgentDisks() ([]DiskInfo, time.Time, bool) {+ return linkedAgentDisksFromResource(v.r)+}++func linkedAgentDisksFromResource(r *Resource) ([]DiskInfo, time.Time, bool) {+ if r == nil || r.Agent == nil || r.Agent.Stale || len(r.Agent.Disks) == 0 {+ return nil, time.Time{}, false+ }+ status, ok := r.SourceStatus[SourceAgent]+ // The registry owns configured lease expiry. A live platform source alone+ // cannot make old, undated or future agent evidence current.+ if !ok || status.Status != "online" || status.LastSeen.IsZero() || status.LastSeen.After(time.Now()) {+ return nil, time.Time{}, false+ }+ return cloneDiskInfos(r.Agent.Disks), status.LastSeen, true+}+// linkedAgentMemoryFromResource reads the agent-owned memory sample attached to// a merged guest resource. Correlation removes the standalone host row, so the// guest view is the only place the agent sample survives (#1962, #2148).@@ -420,6 +439,10 @@ func (v ContainerView) LinkedAgentMemory() (models.Memory, bool) {return linkedAgentMemoryFromResource(v.r)}+func (v ContainerView) LinkedAgentDisks() ([]DiskInfo, time.Time, bool) {+ return linkedAgentDisksFromResource(v.r)+}+func (v ContainerView) String() string { return fmt.Sprintf("ContainerView(%s, %q)", v.ID(), v.Name()) }func (v ContainerView) ID() string {@@ -1201,6 +1224,12 @@ func (v HostView) Disks() []DiskInfo {return cloneDiskInfos(v.r.Agent.Disks)}+// CurrentAgentDisks excludes retained inventory whose source stopped reporting;+// Disks still exposes that inventory for last-known presentation.+func (v HostView) CurrentAgentDisks() ([]DiskInfo, time.Time, bool) {+ return linkedAgentDisksFromResource(v.r)+}+func (v HostView) LinkedNodeID() string {if v.r == nil || v.r.Agent == nil {return ""