Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Keep unreported chart fallbacks empty after lease expiry

rcourtmanddaedc8c7838
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index eb01f3116a..3b829b8dea 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -9894,7 +9894,10 @@ separate from the monitoring reporting lease). A saved enrollment awaiting its
first report in the current monitor generation retains pending delivery and
its historical receipt under the monitoring-owned lease. Connections must not
label that enrollment active, or turn an explicitly expired lease active using
-the longer heartbeat fallback. Agent registration, enrolment,
+the longer heartbeat fallback. Crossing that deadline also cannot create a
+live summary-chart sample from absent telemetry: pending and offline
+enrollments retain stored history without supplying current fallback values.
+Agent registration, enrolment,
install, update, removal and report identity are unchanged.
### Mock-mode read views fold linked fixture agents
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md
index 70922de20c..abfe9702f0 100644
--- a/docs/release-control/v6/internal/subsystems/api-contracts.md
+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md
@@ -21,10 +21,11 @@
## Purpose
-### Pending agent delivery supplies no live chart fallback
+### Unreported agent delivery supplies no live chart fallback
-A saved enrollment awaiting its first accepted report remains queryable by its
-existing metrics target. History and summary chart routes retain stored series
+A saved enrollment awaiting its first accepted report, including after its
+reporting lease expires, remains queryable by its existing metrics target.
+History and summary chart routes retain stored series
but cannot synthesize a now-stamped live sample from that enrollment's absent
metrics. An accepted report restores normal live fallback behaviour.
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index c288127e96..b1fc06758a 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -25,7 +25,9 @@
An agent's pending first report after a server restart is a monitoring-owned
delivery state. Its retained enrollment and historical receipt carry no fresh
-storage metrics, healthy storage verdict or recovery evidence. Storage and
+storage metrics, healthy storage verdict or recovery evidence. Expiry of that
+reporting lease likewise supplies no new zero-valued chart observation or
+recovery proof. Storage and
recovery consumers continue to require their own source observations.
### Failed posture refreshes withdraw retained assessments (24 October 2026)
diff --git a/frontend-modern/src/components/Settings/__tests__/connectionsTableModel.branchcov.test.ts b/frontend-modern/src/components/Settings/__tests__/connectionsTableModel.branchcov.test.ts
index aa90ea5985..861ad50993 100644
--- a/frontend-modern/src/components/Settings/__tests__/connectionsTableModel.branchcov.test.ts
+++ b/frontend-modern/src/components/Settings/__tests__/connectionsTableModel.branchcov.test.ts
@@ -174,7 +174,7 @@ describe('agentAttachmentProblem', () => {
'critical',
],
['unauthorized', 'Agent unauthorized', 'The Pulse Agent token is being rejected.', 'critical'],
- ['pending', 'Agent pending first report', 'The Pulse Agent has not reported yet.', 'warning'],
+ ['pending', 'Waiting for first report', 'The Pulse Agent has not reported yet.', 'warning'],
] as const satisfies ReadonlyArray<[ConnectionState, string, string, 'critical' | 'warning']>)(
'uses the fallback detail for state %s when stateReason is empty',
(state, label, detail, tone) => {
@@ -190,7 +190,7 @@ describe('agentAttachmentProblem', () => {
it.each([
['unreachable', 'Agent unreachable'],
['unauthorized', 'Agent unauthorized'],
- ['pending', 'Agent pending first report'],
+ ['pending', 'Waiting for first report'],
] as const satisfies ReadonlyArray<[ConnectionState, string]>)(
'uses the explicit stateReason for state %s when one is provided',
(state, label) => {
diff --git a/internal/api/chartapi/service.go b/internal/api/chartapi/service.go
index 26a14f2e61..af1e6be4a3 100644
--- a/internal/api/chartapi/service.go
+++ b/internal/api/chartapi/service.go
@@ -362,7 +362,7 @@ func (r *Service) HandleCharts(w http.ResponseWriter, req *http.Request) {
if batchMetrics, ok := agentBatchMetrics[request.SQLResourceID]; ok {
oldestTimestamp = fillChartSeriesFromBatch(agentData[hID], batchMetrics, oldestTimestamp)
}
- if len(agentData[hID]["cpu"]) == 0 && !hostAgentAwaitingFirstReport(h) {
+ if len(agentData[hID]["cpu"]) == 0 && hostAgentCanSupplyLiveChartFallback(h) {
agentData[hID]["cpu"] = []MetricPoint{{Timestamp: currentTime, Value: h.CPUPercent()}}
agentData[hID]["memory"] = []MetricPoint{{Timestamp: currentTime, Value: h.MemoryPercent()}}
agentData[hID]["disk"] = []MetricPoint{{Timestamp: currentTime, Value: h.DiskPercent()}}
@@ -478,9 +478,12 @@ func parseWorkloadMaxPoints(raw string) int {
func ParseWorkloadMaxPoints(raw string) int { return parseWorkloadMaxPoints(raw) }
-func hostAgentAwaitingFirstReport(host *unifiedresources.HostView) bool {
+func hostAgentCanSupplyLiveChartFallback(host *unifiedresources.HostView) bool {
+ if host.Status() == unifiedresources.StatusOffline {
+ return false
+ }
status, ok := host.SourceStatus(unifiedresources.SourceAgent)
- return ok && status.Status == "pending"
+ return !ok || status.Status != "pending"
}
func hostAgentChartRequest(host *unifiedresources.HostView) (string, monitoring.GuestChartRequest, bool) {
@@ -1964,7 +1967,7 @@ func (r *Service) HandleInfrastructureCharts(w http.ResponseWriter, req *http.Re
if len(agentData[hID][metricType]) > 0 {
continue
}
- if hostAgentAwaitingFirstReport(h) {
+ if !hostAgentCanSupplyLiveChartFallback(h) {
continue
}
var value float64
diff --git a/internal/api/chartapi/service_handlers_test.go b/internal/api/chartapi/service_handlers_test.go
index 2d7a71818b..27c57f422a 100644
--- a/internal/api/chartapi/service_handlers_test.go
+++ b/internal/api/chartapi/service_handlers_test.go
@@ -239,47 +239,51 @@ func TestHandleCharts_UsesCanonicalMockUnifiedReadStateForVMwareHosts(t *testing
}
func TestChartsDoNotSynthesizeCurrentMetricsForPendingEnrollment(t *testing.T) {
- monitor, state, history := newTestMonitor(t)
- syncTestResourceStore(t, monitor, state)
- continuity := config.NewHostContinuityStore(t.TempDir(), nil)
- if err := continuity.Upsert(config.HostContinuityEntry{HostID: "waiting-agent", Hostname: "waiting", LastSeen: time.Now().Add(-time.Second), IntervalSeconds: 30}); err != nil {
- t.Fatal(err)
- }
- setUnexportedField(t, monitor, "hostContinuityStore", continuity)
- for _, stored := range []bool{false, true} {
- if stored {
- history.AddGuestMetric("agent:waiting-agent", "cpu", 37, time.Now().Add(-time.Minute))
- }
- for _, path := range []string{"/api/charts?range=5m", "/api/charts/infrastructure?range=5m"} {
- service := NewService(fixedMonitorResolver{monitor: monitor})
- rec := httptest.NewRecorder()
- req := httptest.NewRequest(http.MethodGet, path, nil)
- if strings.HasPrefix(path, "/api/charts/infrastructure") {
- service.HandleInfrastructureCharts(rec, req)
- } else {
- service.HandleCharts(rec, req)
- }
- var response struct {
- AgentData map[string]VMChartData `json:"agentData"`
- }
- if rec.Code != http.StatusOK {
- t.Fatalf("%s: status %d: %s", path, rec.Code, rec.Body.String())
- }
- if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
+ for _, age := range []time.Duration{time.Second, 4 * time.Minute} {
+ t.Run(age.String(), func(t *testing.T) {
+ monitor, state, history := newTestMonitor(t)
+ syncTestResourceStore(t, monitor, state)
+ continuity := config.NewHostContinuityStore(t.TempDir(), nil)
+ if err := continuity.Upsert(config.HostContinuityEntry{HostID: "waiting-agent", Hostname: "waiting", LastSeen: time.Now().Add(-age), IntervalSeconds: 30}); err != nil {
t.Fatal(err)
}
- series := response.AgentData["waiting-agent"]
- if _, ok := response.AgentData["waiting-agent"]; !ok {
- t.Fatalf("%s lost the pending enrollment", path)
- }
- if stored {
- if len(series["cpu"]) != 1 || series["cpu"][0].Value != 37 {
- t.Fatalf("%s lost stored history: %+v", path, series)
+ setUnexportedField(t, monitor, "hostContinuityStore", continuity)
+ for _, stored := range []bool{false, true} {
+ if stored {
+ history.AddGuestMetric("agent:waiting-agent", "cpu", 37, time.Now().Add(-4*time.Minute))
+ }
+ for _, path := range []string{"/api/charts?range=5m", "/api/charts/infrastructure?range=5m"} {
+ service := NewService(fixedMonitorResolver{monitor: monitor})
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, path, nil)
+ if strings.HasPrefix(path, "/api/charts/infrastructure") {
+ service.HandleInfrastructureCharts(rec, req)
+ } else {
+ service.HandleCharts(rec, req)
+ }
+ var response struct {
+ AgentData map[string]VMChartData `json:"agentData"`
+ }
+ if rec.Code != http.StatusOK {
+ t.Fatalf("%s: status %d: %s", path, rec.Code, rec.Body.String())
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
+ t.Fatal(err)
+ }
+ series := response.AgentData["waiting-agent"]
+ if _, ok := response.AgentData["waiting-agent"]; !ok {
+ t.Fatalf("%s lost the pending enrollment", path)
+ }
+ if stored {
+ if len(series["cpu"]) != 1 || series["cpu"][0].Value != 37 {
+ t.Fatalf("%s lost stored history: %+v", path, series)
+ }
+ } else if len(series) != 0 {
+ t.Fatalf("%s invented current readings: %+v", path, series)
+ }
}
- } else if len(series) != 0 {
- t.Fatalf("%s invented current readings: %+v", path, series)
}
- }
+ })
}
}
diff --git a/internal/api/connections_handlers_continuity_test.go b/internal/api/connections_handlers_continuity_test.go
index e1b78cb1e9..69515d36b8 100644
--- a/internal/api/connections_handlers_continuity_test.go
+++ b/internal/api/connections_handlers_continuity_test.go
@@ -24,7 +24,7 @@ func TestConnectionsHandleListIncludesContinuityBackedHostAgents(t *testing.T) {
}
t.Cleanup(seedMonitor.Stop)
- _, err = seedMonitor.ApplyHostReport(agentshost.Report{
+ seedHost, err := seedMonitor.ApplyHostReport(agentshost.Report{
Agent: agentshost.AgentInfo{
ID: "agent-1",
Version: "6.0.0-rc.1",
@@ -79,14 +79,14 @@ func TestConnectionsHandleListIncludesContinuityBackedHostAgents(t *testing.T) {
if conn.Name != "host-1.local" || conn.Address != "host-1.local" {
t.Fatalf("unexpected continuity-backed connection identity: name=%q address=%q", conn.Name, conn.Address)
}
- if conn.State != ConnectionStateActive {
- t.Fatalf("connection state = %q, want %q", conn.State, ConnectionStateActive)
+ if conn.State != ConnectionStatePending || conn.StateReason != "Waiting for first report after server restart" {
+ t.Fatalf("restarted connection = %q (%q), want pending first report", conn.State, conn.StateReason)
}
if conn.Source != ConnectionSourceAgent {
t.Fatalf("connection source = %q, want %q", conn.Source, ConnectionSourceAgent)
}
- if conn.LastSeen == nil || conn.LastSeen.IsZero() {
- t.Fatalf("expected continuity-backed connection to carry lastSeen, got %#v", conn.LastSeen)
+ if conn.LastSeen == nil || !conn.LastSeen.Equal(seedHost.LastSeen) {
+ t.Fatalf("expected continuity-backed connection to preserve lastSeen %s, got %#v", seedHost.LastSeen, conn.LastSeen)
}
}
diff --git a/internal/api/contract_test.go b/internal/api/contract_test.go
index 4678e26bf9..8e8b37dd94 100644
--- a/internal/api/contract_test.go
+++ b/internal/api/contract_test.go
@@ -13940,8 +13940,13 @@ func TestContract_MetricsHistoryLivePointOnlyWhileSourceReports(t *testing.T) {
setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(10, time.Hour))
setUnexportedField(t, monitor, "metricsStore", store)
continuity := config.NewHostContinuityStore(t.TempDir(), nil)
- if err := continuity.Upsert(config.HostContinuityEntry{HostID: "agent-waiting", Hostname: "waiting", LastSeen: now.Add(-time.Second), IntervalSeconds: 30}); err != nil {
- t.Fatal(err)
+ for _, entry := range []config.HostContinuityEntry{
+ {HostID: "agent-waiting", Hostname: "waiting", LastSeen: now.Add(-time.Second), IntervalSeconds: 30},
+ {HostID: "agent-expired-waiting", Hostname: "expired-waiting", LastSeen: now.Add(-4 * time.Minute), IntervalSeconds: 30},
+ } {
+ if err := continuity.Upsert(entry); err != nil {
+ t.Fatal(err)
+ }
}
setUnexportedField(t, monitor, "hostContinuityStore", continuity)
router := &Router{monitor: monitor}
@@ -13983,6 +13988,8 @@ func TestContract_MetricsHistoryLivePointOnlyWhileSourceReports(t *testing.T) {
for _, tc := range []struct{ resourceType, resourceID, metric string }{
{"agent", "agent-waiting", "cpu"},
{"agent", "agent-waiting", "temperature"},
+ {"agent", "agent-expired-waiting", "cpu"},
+ {"agent", "agent-expired-waiting", "temperature"},
{"agent", "agent-silent", "cpu"},
{"agent", "agent-silent", "temperature"},
{"node", "pve-unreachable", "memory"},
@@ -14039,11 +14046,58 @@ func TestContract_MetricsHistoryLivePointOnlyWhileSourceReports(t *testing.T) {
t.Fatalf("%s %s: current %s not served as a live point: source=%q points=%+v", tc.resourceType, tc.resourceID, tc.metric, resp.Source, resp.Points)
}
}
+ // Summary endpoints share the same reading contract on both sides of the
+ // original lease. An expired saved enrollment has no new zero reading.
+ assertSavedSummary := func(stored bool) {
+ t.Helper()
+ for _, path := range []string{"/api/charts?range=5m", "/api/charts/infrastructure?range=5m"} {
+ // Each request gets a fresh service so the summary cache cannot hide
+ // newly added history in the second assertion.
+ chartsRouter := &Router{monitor: monitor}
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, path, nil)
+ if strings.HasPrefix(path, "/api/charts/infrastructure") {
+ chartsRouter.handleInfrastructureCharts(rec, req)
+ } else {
+ chartsRouter.handleCharts(rec, req)
+ }
+ if rec.Code != http.StatusOK {
+ t.Fatalf("%s: status = %d, body=%s", path, rec.Code, rec.Body.String())
+ }
+ var response struct {
+ AgentData map[string]chartapi.VMChartData `json:"agentData"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil {
+ t.Fatalf("%s: decode: %v", path, err)
+ }
+ for _, id := range []string{"agent-waiting", "agent-expired-waiting"} {
+ series, ok := response.AgentData[id]
+ if !ok {
+ t.Fatalf("%s lost saved enrollment %s", path, id)
+ }
+ if stored && id == "agent-waiting" {
+ if len(series["cpu"]) != 1 || series["cpu"][0].Value != 37 {
+ t.Fatalf("%s %s: stored history changed: %+v", path, id, series)
+ }
+ }
+ for metric, points := range series {
+ if stored && id == "agent-waiting" && metric == "cpu" {
+ continue
+ }
+ if len(points) != 0 {
+ t.Fatalf("%s %s: saved enrollment served invented %s readings: %+v", path, id, metric, points)
+ }
+ }
+ }
+ }
+ }
+ assertSavedSummary(false)
// A saved enrollment may still have real history from its previous run.
monitor.GetMetricsHistory().AddGuestMetric("agent:agent-waiting", "cpu", 37, now.Add(-time.Minute))
if resp, _ := history("resourceType=agent&resourceId=agent-waiting&metric=cpu"); resp.Source == "live" || len(resp.Points) != 1 || resp.Points[0].Value != 37 {
t.Fatalf("pending enrollment lost stored history or relabelled it as live: %+v", resp)
}
+ assertSavedSummary(true)
}
// A row lapses only when it records a reading source and every one it records
diff --git a/internal/api/resources_link_continuity_test.go b/internal/api/resources_link_continuity_test.go
index 031102797c..92e1bf4e0e 100644
--- a/internal/api/resources_link_continuity_test.go
+++ b/internal/api/resources_link_continuity_test.go
@@ -16,8 +16,8 @@ import (
// one identity, so the resources API stores operator state written through the
// agent's references on the VM and Patrol reads it there, on the same store.
// Neither the monitor's read state nor the registry the resources API seeds
-// from it merges the saved agent into the VM: the agent keeps its own offline
-// row and the VM keeps what vSphere reports.
+// from it merges the saved agent into the VM: the recent agent keeps its own
+// pending row without current metrics and the VM keeps what vSphere reports.
func TestResourcesAPIAndPatrolAgreeOnContinuityAgentLinkedIntoGuest(t *testing.T) {
now := time.Now().UTC()
monitorCfg := &config.Config{DataPath: t.TempDir()}
@@ -85,8 +85,8 @@ func TestResourcesAPIAndPatrolAgreeOnContinuityAgentLinkedIntoGuest(t *testing.T
for _, resource := range listing {
readStateRows[resource.ID] = resource
}
- if agent, ok := readStateRows[agentID]; !ok || agent.Status != unifiedresources.StatusOffline {
- t.Fatalf("read state agent row = %+v (listed=%v), want the saved agent's own offline row", agent, ok)
+ if agent, ok := readStateRows[agentID]; !ok || agent.Status != unifiedresources.StatusUnknown || agent.SourceStatus[unifiedresources.SourceAgent].Status != "pending" || agent.Metrics != nil {
+ t.Fatalf("read state agent row = %+v (listed=%v), want the saved agent's own pending row without current metrics", agent, ok)
}
if guest, ok := readStateRows[vmID]; !ok || guest.Agent != nil {
t.Fatalf("read state guest = %+v (listed=%v), want vSphere's VM without the saved agent", guest, ok)
@@ -97,8 +97,8 @@ func TestResourcesAPIAndPatrolAgreeOnContinuityAgentLinkedIntoGuest(t *testing.T
t.Fatalf("resources API registry: %v", err)
}
agent, listed := registry.Get(agentID)
- if !listed || agent.Status != unifiedresources.StatusOffline {
- t.Fatalf("resources API agent row = %+v (listed=%v), want the saved agent's own offline row", agent, listed)
+ if !listed || agent.Status != unifiedresources.StatusUnknown || agent.SourceStatus[unifiedresources.SourceAgent].Status != "pending" || agent.Metrics != nil {
+ t.Fatalf("resources API agent row = %+v (listed=%v), want the saved agent's own pending row without current metrics", agent, listed)
}
if guest, ok := registry.Get(vmID); !ok || guest.Agent != nil || guest.Status != unifiedresources.StatusOnline {
t.Fatalf("resources API guest = %+v (listed=%v), want vSphere's online VM without the saved agent", guest, ok)