Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep unreported chart fallbacks empty after lease expiry
rcourtmanddaedc8c7838
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.mdindex eb01f3116a..3b829b8dea 100644--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md@@ -9894,7 +9894,10 @@ separate from the monitoring reporting lease). A saved enrollment awaiting itsfirst report in the current monitor generation retains pending delivery andits historical receipt under the monitoring-owned lease. Connections must notlabel that enrollment active, or turn an explicitly expired lease active using-the longer heartbeat fallback. Agent registration, enrolment,+the longer heartbeat fallback. Crossing that deadline also cannot create a+live summary-chart sample from absent telemetry: pending and offline+enrollments retain stored history without supplying current fallback values.+Agent registration, enrolment,install, update, removal and report identity are unchanged.### Mock-mode read views fold linked fixture agentsdiff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.mdindex 70922de20c..abfe9702f0 100644--- a/docs/release-control/v6/internal/subsystems/api-contracts.md+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md@@ -21,10 +21,11 @@## Purpose-### Pending agent delivery supplies no live chart fallback+### Unreported agent delivery supplies no live chart fallback-A saved enrollment awaiting its first accepted report remains queryable by its-existing metrics target. History and summary chart routes retain stored series+A saved enrollment awaiting its first accepted report, including after its+reporting lease expires, remains queryable by its existing metrics target.+History and summary chart routes retain stored seriesbut cannot synthesize a now-stamped live sample from that enrollment's absentmetrics. An accepted report restores normal live fallback behaviour.diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex c288127e96..b1fc06758a 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -25,7 +25,9 @@An agent's pending first report after a server restart is a monitoring-owneddelivery state. Its retained enrollment and historical receipt carry no fresh-storage metrics, healthy storage verdict or recovery evidence. Storage and+storage metrics, healthy storage verdict or recovery evidence. Expiry of that+reporting lease likewise supplies no new zero-valued chart observation or+recovery proof. Storage andrecovery consumers continue to require their own source observations.### Failed posture refreshes withdraw retained assessments (24 October 2026)diff --git a/frontend-modern/src/components/Settings/__tests__/connectionsTableModel.branchcov.test.ts b/frontend-modern/src/components/Settings/__tests__/connectionsTableModel.branchcov.test.tsindex aa90ea5985..861ad50993 100644--- a/frontend-modern/src/components/Settings/__tests__/connectionsTableModel.branchcov.test.ts+++ b/frontend-modern/src/components/Settings/__tests__/connectionsTableModel.branchcov.test.ts@@ -174,7 +174,7 @@ describe('agentAttachmentProblem', () => {'critical',],['unauthorized', 'Agent unauthorized', 'The Pulse Agent token is being rejected.', 'critical'],- ['pending', 'Agent pending first report', 'The Pulse Agent has not reported yet.', 'warning'],+ ['pending', 'Waiting for first report', 'The Pulse Agent has not reported yet.', 'warning'],] as const satisfies ReadonlyArray<[ConnectionState, string, string, 'critical' | 'warning']>)('uses the fallback detail for state %s when stateReason is empty',(state, label, detail, tone) => {@@ -190,7 +190,7 @@ describe('agentAttachmentProblem', () => {it.each([['unreachable', 'Agent unreachable'],['unauthorized', 'Agent unauthorized'],- ['pending', 'Agent pending first report'],+ ['pending', 'Waiting for first report'],] as const satisfies ReadonlyArray<[ConnectionState, string]>)('uses the explicit stateReason for state %s when one is provided',(state, label) => {diff --git a/internal/api/chartapi/service.go b/internal/api/chartapi/service.goindex 26a14f2e61..af1e6be4a3 100644--- a/internal/api/chartapi/service.go+++ b/internal/api/chartapi/service.go@@ -362,7 +362,7 @@ func (r *Service) HandleCharts(w http.ResponseWriter, req *http.Request) {if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok {oldestTimestamp = fillChartSeriesFromBatch(agentData[hID], batchMetrics, oldestTimestamp)}- if len(agentData[hID]["cpu"]) == 0 && !hostAgentAwaitingFirstReport(h) {+ if len(agentData[hID]["cpu"]) == 0 && hostAgentCanSupplyLiveChartFallback(h) {agentData[hID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: h.CPUPercent()}}agentData[hID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: h.MemoryPercent()}}agentData[hID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: h.DiskPercent()}}@@ -478,9 +478,12 @@ func parseWorkloadMaxPoints(raw string) int {func ParseWorkloadMaxPoints(raw string) int { return parseWorkloadMaxPoints(raw) }-func hostAgentAwaitingFirstReport(host *unifiedresources.HostView) bool {+func hostAgentCanSupplyLiveChartFallback(host *unifiedresources.HostView) bool {+ if host.Status() == unifiedresources.StatusOffline {+ return false+ }status, ok := host.SourceStatus(unifiedresources.SourceAgent)- return ok && status.Status == "pending"+ return !ok || status.Status != "pending"}func hostAgentChartRequest(host *unifiedresources.HostView) (string, monitoring.GuestChartRequest, bool) {@@ -1964,7 +1967,7 @@ func (r *Service) HandleInfrastructureCharts(w http.ResponseWriter, req *http.Reif len(agentData[hID][metricType]) > 0 {continue}- if hostAgentAwaitingFirstReport(h) {+ if !hostAgentCanSupplyLiveChartFallback(h) {continue}var value float64diff --git a/internal/api/chartapi/service_handlers_test.go b/internal/api/chartapi/service_handlers_test.goindex 2d7a71818b..27c57f422a 100644--- a/internal/api/chartapi/service_handlers_test.go+++ b/internal/api/chartapi/service_handlers_test.go@@ -239,47 +239,51 @@ func TestHandleCharts_UsesCanonicalMockUnifiedReadStateForVMwareHosts(t *testing}func TestChartsDoNotSynthesizeCurrentMetricsForPendingEnrollment(t *testing.T) {- monitor, state, history := newTestMonitor(t)- syncTestResourceStore(t, monitor, state)- continuity := config.NewHostContinuityStore(t.TempDir(), nil)- if err := continuity.Upsert(config.HostContinuityEntry{HostID: "waiting-agent", Hostname: "waiting", LastSeen: time.Now().Add(-time.Second), IntervalSeconds: 30}); err != nil {- t.Fatal(err)- }- setUnexportedField(t, monitor, "hostContinuityStore", continuity)- for _, stored := range []bool{false, true} {- if stored {- history.AddGuestMetric("agent:waiting-agent", "cpu", 37, time.Now().Add(-time.Minute))- }- for _, path := range []string{"/api/charts?range=5m", "/api/charts/infrastructure?range=5m"} {- service := NewService(fixedMonitorResolver{monitor: monitor})- rec := httptest.NewRecorder()- req := httptest.NewRequest(http.MethodGet, path, nil)- if strings.HasPrefix(path, "/api/charts/infrastructure") {- service.HandleInfrastructureCharts(rec, req)- } else {- service.HandleCharts(rec, req)- }- var response struct {- AgentData map[string]VMChartData `json:"agentData"`- }- if rec.Code != http.StatusOK {- t.Fatalf("%s: status %d: %s", path, rec.Code, rec.Body.String())- }- if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {+ for _, age := range []time.Duration{time.Second, 4 * time.Minute} {+ t.Run(age.String(), func(t *testing.T) {+ monitor, state, history := newTestMonitor(t)+ syncTestResourceStore(t, monitor, state)+ continuity := config.NewHostContinuityStore(t.TempDir(), nil)+ if err := continuity.Upsert(config.HostContinuityEntry{HostID: "waiting-agent", Hostname: "waiting", LastSeen: time.Now().Add(-age), IntervalSeconds: 30}); err != nil {t.Fatal(err)}- series := response.AgentData["waiting-agent"]- if _, ok := response.AgentData["waiting-agent"]; !ok {- t.Fatalf("%s lost the pending enrollment", path)- }- if stored {- if len(series["cpu"]) != 1 || series["cpu"][0].Value != 37 {- t.Fatalf("%s lost stored history: %+v", path, series)+ setUnexportedField(t, monitor, "hostContinuityStore", continuity)+ for _, stored := range []bool{false, true} {+ if stored {+ history.AddGuestMetric("agent:waiting-agent", "cpu", 37, time.Now().Add(-4*time.Minute))+ }+ for _, path := range []string{"/api/charts?range=5m", "/api/charts/infrastructure?range=5m"} {+ service := NewService(fixedMonitorResolver{monitor: monitor})+ rec := httptest.NewRecorder()+ req := httptest.NewRequest(http.MethodGet, path, nil)+ if strings.HasPrefix(path, "/api/charts/infrastructure") {+ service.HandleInfrastructureCharts(rec, req)+ } else {+ service.HandleCharts(rec, req)+ }+ var response struct {+ AgentData map[string]VMChartData `json:"agentData"`+ }+ if rec.Code != http.StatusOK {+ t.Fatalf("%s: status %d: %s", path, rec.Code, rec.Body.String())+ }+ if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {+ t.Fatal(err)+ }+ series := response.AgentData["waiting-agent"]+ if _, ok := response.AgentData["waiting-agent"]; !ok {+ t.Fatalf("%s lost the pending enrollment", path)+ }+ if stored {+ if len(series["cpu"]) != 1 || series["cpu"][0].Value != 37 {+ t.Fatalf("%s lost stored history: %+v", path, series)+ }+ } else if len(series) != 0 {+ t.Fatalf("%s invented current readings: %+v", path, series)+ }}- } else if len(series) != 0 {- t.Fatalf("%s invented current readings: %+v", path, series)}- }+ })}}diff --git a/internal/api/connections_handlers_continuity_test.go b/internal/api/connections_handlers_continuity_test.goindex e1b78cb1e9..69515d36b8 100644--- a/internal/api/connections_handlers_continuity_test.go+++ b/internal/api/connections_handlers_continuity_test.go@@ -24,7 +24,7 @@ func TestConnectionsHandleListIncludesContinuityBackedHostAgents(t *testing.T) {}t.Cleanup(seedMonitor.Stop)- _, err = seedMonitor.ApplyHostReport(agentshost.Report{+ seedHost, err := seedMonitor.ApplyHostReport(agentshost.Report{Agent: agentshost.AgentInfo{ID: "agent-1",Version: "6.0.0-rc.1",@@ -79,14 +79,14 @@ func TestConnectionsHandleListIncludesContinuityBackedHostAgents(t *testing.T) {if conn.Name != "host-1.local" || conn.Address != "host-1.local" {t.Fatalf("unexpected continuity-backed connection identity: name=%q address=%q", conn.Name, conn.Address)}- if conn.State != ConnectionStateActive {- t.Fatalf("connection state = %q, want %q", conn.State, ConnectionStateActive)+ if conn.State != ConnectionStatePending || conn.StateReason != "Waiting for first report after server restart" {+ t.Fatalf("restarted connection = %q (%q), want pending first report", conn.State, conn.StateReason)}if conn.Source != ConnectionSourceAgent {t.Fatalf("connection source = %q, want %q", conn.Source, ConnectionSourceAgent)}- if conn.LastSeen == nil || conn.LastSeen.IsZero() {- t.Fatalf("expected continuity-backed connection to carry lastSeen, got %#v", conn.LastSeen)+ if conn.LastSeen == nil || !conn.LastSeen.Equal(seedHost.LastSeen) {+ t.Fatalf("expected continuity-backed connection to preserve lastSeen %s, got %#v", seedHost.LastSeen, conn.LastSeen)}}diff --git a/internal/api/contract_test.go b/internal/api/contract_test.goindex 4678e26bf9..8e8b37dd94 100644--- a/internal/api/contract_test.go+++ b/internal/api/contract_test.go@@ -13940,8 +13940,13 @@ func TestContract_MetricsHistoryLivePointOnlyWhileSourceReports(t *testing.T) {setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(10, time.Hour))setUnexportedField(t, monitor, "metricsStore", store)continuity := config.NewHostContinuityStore(t.TempDir(), nil)- if err := continuity.Upsert(config.HostContinuityEntry{HostID: "agent-waiting", Hostname: "waiting", LastSeen: now.Add(-time.Second), IntervalSeconds: 30}); err != nil {- t.Fatal(err)+ for _, entry := range []config.HostContinuityEntry{+ {HostID: "agent-waiting", Hostname: "waiting", LastSeen: now.Add(-time.Second), IntervalSeconds: 30},+ {HostID: "agent-expired-waiting", Hostname: "expired-waiting", LastSeen: now.Add(-4 * time.Minute), IntervalSeconds: 30},+ } {+ if err := continuity.Upsert(entry); err != nil {+ t.Fatal(err)+ }}setUnexportedField(t, monitor, "hostContinuityStore", continuity)router := &Router{monitor: monitor}@@ -13983,6 +13988,8 @@ func TestContract_MetricsHistoryLivePointOnlyWhileSourceReports(t *testing.T) {for _, tc := range []struct{ resourceType, resourceID, metric string }{{"agent", "agent-waiting", "cpu"},{"agent", "agent-waiting", "temperature"},+ {"agent", "agent-expired-waiting", "cpu"},+ {"agent", "agent-expired-waiting", "temperature"},{"agent", "agent-silent", "cpu"},{"agent", "agent-silent", "temperature"},{"node", "pve-unreachable", "memory"},@@ -14039,11 +14046,58 @@ func TestContract_MetricsHistoryLivePointOnlyWhileSourceReports(t *testing.T) {t.Fatalf("%s %s: current %s not served as a live point: source=%q points=%+v", tc.resourceType, tc.resourceID, tc.metric, resp.Source, resp.Points)}}+ // Summary endpoints share the same reading contract on both sides of the+ // original lease. An expired saved enrollment has no new zero reading.+ assertSavedSummary := func(stored bool) {+ t.Helper()+ for _, path := range []string{"/api/charts?range=5m", "/api/charts/infrastructure?range=5m"} {+ // Each request gets a fresh service so the summary cache cannot hide+ // newly added history in the second assertion.+ chartsRouter := &Router{monitor: monitor}+ rec := httptest.NewRecorder()+ req := httptest.NewRequest(http.MethodGet, path, nil)+ if strings.HasPrefix(path, "/api/charts/infrastructure") {+ chartsRouter.handleInfrastructureCharts(rec, req)+ } else {+ chartsRouter.handleCharts(rec, req)+ }+ if rec.Code != http.StatusOK {+ t.Fatalf("%s: status = %d, body=%s", path, rec.Code, rec.Body.String())+ }+ var response struct {+ AgentData map[string]chartapi.VMChartData `json:"agentData"`+ }+ if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {+ t.Fatalf("%s: decode: %v", path, err)+ }+ for _, id := range []string{"agent-waiting", "agent-expired-waiting"} {+ series, ok := response.AgentData[id]+ if !ok {+ t.Fatalf("%s lost saved enrollment %s", path, id)+ }+ if stored && id == "agent-waiting" {+ if len(series["cpu"]) != 1 || series["cpu"][0].Value != 37 {+ t.Fatalf("%s %s: stored history changed: %+v", path, id, series)+ }+ }+ for metric, points := range series {+ if stored && id == "agent-waiting" && metric == "cpu" {+ continue+ }+ if len(points) != 0 {+ t.Fatalf("%s %s: saved enrollment served invented %s readings: %+v", path, id, metric, points)+ }+ }+ }+ }+ }+ assertSavedSummary(false)// A saved enrollment may still have real history from its previous run.monitor.GetMetricsHistory().AddGuestMetric("agent:agent-waiting", "cpu", 37, now.Add(-time.Minute))if resp, _ := history("resourceType=agent&resourceId=agent-waiting&metric=cpu"); resp.Source == "live" || len(resp.Points) != 1 || resp.Points[0].Value != 37 {t.Fatalf("pending enrollment lost stored history or relabelled it as live: %+v", resp)}+ assertSavedSummary(true)}// A row lapses only when it records a reading source and every one it recordsdiff --git a/internal/api/resources_link_continuity_test.go b/internal/api/resources_link_continuity_test.goindex 031102797c..92e1bf4e0e 100644--- a/internal/api/resources_link_continuity_test.go+++ b/internal/api/resources_link_continuity_test.go@@ -16,8 +16,8 @@ import (// one identity, so the resources API stores operator state written through the// agent's references on the VM and Patrol reads it there, on the same store.// Neither the monitor's read state nor the registry the resources API seeds-// from it merges the saved agent into the VM: the agent keeps its own offline-// row and the VM keeps what vSphere reports.+// from it merges the saved agent into the VM: the recent agent keeps its own+// pending row without current metrics and the VM keeps what vSphere reports.func TestResourcesAPIAndPatrolAgreeOnContinuityAgentLinkedIntoGuest(t *testing.T) {now := time.Now().UTC()monitorCfg := &config.Config{DataPath: t.TempDir()}@@ -85,8 +85,8 @@ func TestResourcesAPIAndPatrolAgreeOnContinuityAgentLinkedIntoGuest(t *testing.Tfor _, resource := range listing {readStateRows[resource.ID] = resource}- if agent, ok := readStateRows[agentID]; !ok || agent.Status != unifiedresources.StatusOffline {- t.Fatalf("read state agent row = %+v (listed=%v), want the saved agent's own offline row", agent, ok)+ if agent, ok := readStateRows[agentID]; !ok || agent.Status != unifiedresources.StatusUnknown || agent.SourceStatus[unifiedresources.SourceAgent].Status != "pending" || agent.Metrics != nil {+ t.Fatalf("read state agent row = %+v (listed=%v), want the saved agent's own pending row without current metrics", agent, ok)}if guest, ok := readStateRows[vmID]; !ok || guest.Agent != nil {t.Fatalf("read state guest = %+v (listed=%v), want vSphere's VM without the saved agent", guest, ok)@@ -97,8 +97,8 @@ func TestResourcesAPIAndPatrolAgreeOnContinuityAgentLinkedIntoGuest(t *testing.Tt.Fatalf("resources API registry: %v", err)}agent, listed := registry.Get(agentID)- if !listed || agent.Status != unifiedresources.StatusOffline {- t.Fatalf("resources API agent row = %+v (listed=%v), want the saved agent's own offline row", agent, listed)+ if !listed || agent.Status != unifiedresources.StatusUnknown || agent.SourceStatus[unifiedresources.SourceAgent].Status != "pending" || agent.Metrics != nil {+ t.Fatalf("resources API agent row = %+v (listed=%v), want the saved agent's own pending row without current metrics", agent, listed)}if guest, ok := registry.Get(vmID); !ok || guest.Agent != nil || guest.Status != unifiedresources.StatusOnline {t.Fatalf("resources API guest = %+v (listed=%v), want vSphere's online VM without the saved agent", guest, ok)