Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

fix(ceph): correct manager counts and health projection

rcourtman59bc124755d8
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 653ee593a4..b07948dcb0 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -920,6 +920,17 @@ An explicit zero active count under a known requirement is retained; neither
device-token inference nor a different kernel fallback probe may fill it in.
The collector/report controls exercise both zero-active boundaries.
+### Ceph standby managers count as running
+
+The host agent's Ceph `mgr` service row counts every manager in the mgr map as
+running while `mgrmap.available` holds. A standby is a running `ceph-mgr` that
+beacons the monitors like the active one and is dropped once those beacons stop
+for `mon_mgr_beacon_grace`, so a healthy cluster with one active and two
+standby managers reports 3 of 3 running, not 1 of 3. Without an available
+manager the row keeps zero running. Report shapes, enrollment and command
+authority are unchanged, and monitoring normalizes rows from earlier agents at
+ingest. `internal/hostagent/agent_metrics_test.go` covers the report row.
+
### Continuity-aware broadcast read ownership
Broadcast state takes its single resource list from the current continuity-aware
diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md
index 094f635375..fa0b131c07 100644
--- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md
+++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md
@@ -2548,6 +2548,16 @@ hover and as the accessible name. Grouped platform tables that already name
the group in a shared grouped-table header row drop columns that would repeat
the group's own value on every row.
+### Ceph health projections
+
+`ProxmoxCephTable` uses one health classification for the row, status-filter
+counts and drawer header. Native `HEALTH_ERR` and the server-normalized `ERR`
+both remain critical, including when no manager daemons are reported. A
+missing service tally must not turn explicit critical provider health into a
+healthy table verdict. Mounted regressions in `ProxmoxCephTable.test.tsx` pin
+the native/normalized vocabulary and selected/unselected critical filtering.
+
+
### Retained state in bounded platform windows
`PlatformWindowedRows` and `PlatformWindowedList` keep one keyed renderer owner
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md
index a54f384602..57cb0a6aa3 100644
--- a/docs/release-control/v6/internal/subsystems/monitoring.md
+++ b/docs/release-control/v6/internal/subsystems/monitoring.md
@@ -6124,6 +6124,21 @@ legacy Unraid raw-status normalization at host-agent ingest: when older agents s
must derive the canonical disk status before storage-risk assessment runs so
v5 aggregate counters do not override clearly healthy per-disk state during v6
compatibility operation.
+Host-agent Ceph `mgr` service rows are normalized at the same ingest boundary
+in `internal/monitoring/monitor_helpers.go`. A standby manager is a running
+`ceph-mgr` that the monitors drop once its beacons stop, so while the reported
+mgr map is available every listed manager counts as running, and rows from
+agents that counted only the active manager never show a healthy cluster's
+standbys as stopped. Without an available manager the row keeps the agent's
+zero running. `internal/monitoring/monitor_host_agents_test.go` pins the cluster
+and host rows that ingest stores.
+Ceph health status, message and checks form one observation at the canonical
+state merge in `internal/models/ceph_cluster_identity.go`. An explicit status
+owns its details, including an empty message/check list, so healthy or changed
+reports cannot inherit an older or lower-priority incident. Only an absent
+status permits the existing health continuity fallback. Other topology/count
+fallbacks and source priority remain independent. `internal/models/state_host_test.go`
+checks recovery, source authority and missing-status continuity.
Unraid long-running operation state uses the same authoritative transition
boundary. A non-empty active sync action may carry progress; cancellation,
completion, or idle state is represented by an empty action and normalized
diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json
index 5c60f4a468..e6b5bcaeda 100644
--- a/docs/release-control/v6/internal/subsystems/registry.json
+++ b/docs/release-control/v6/internal/subsystems/registry.json
@@ -6421,6 +6421,7 @@
"internal/mock/integration.go",
"internal/mock/platform_fixtures.go",
"internal/mock/recovery_points.go",
+ "internal/models/ceph_cluster_identity.go",
"internal/models/converters.go",
"internal/models/deepcopy.go",
"internal/models/metrics_types.go",
@@ -6789,6 +6790,7 @@
"label": "runtime report model proof",
"match_prefixes": [],
"match_files": [
+ "internal/models/ceph_cluster_identity.go",
"internal/models/converters.go",
"internal/models/deepcopy.go",
"internal/models/metrics_types.go",
@@ -9040,6 +9042,7 @@
"frontend-modern/src/features/platformPage/__tests__/platformOverviewLayout.guardrails.test.ts",
"frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.test.ts",
"frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupsTable.test.tsx",
+ "frontend-modern/src/features/proxmox/__tests__/ProxmoxCephTable.test.tsx",
"frontend-modern/src/features/proxmox/__tests__/ProxmoxCoverageTable.test.tsx",
"frontend-modern/src/features/proxmox/__tests__/ProxmoxMailGatewayTable.test.ts",
"frontend-modern/src/features/proxmox/__tests__/ProxmoxNodesTable.test.tsx",
diff --git a/docs/release-control/v6/internal/subsystems/unified-resources.md b/docs/release-control/v6/internal/subsystems/unified-resources.md
index 075d411a6c..d8049734f0 100644
--- a/docs/release-control/v6/internal/subsystems/unified-resources.md
+++ b/docs/release-control/v6/internal/subsystems/unified-resources.md
@@ -2166,6 +2166,11 @@ the daemon tally as width shrinks, keeps one five-value phone projection
(cluster, health, OSDs, pools, capacity) whose values each fit whole, and
leaves quorum membership and the FSID to the cluster disclosure, where the
FSID stays searchable and is shown in full.
+The Ceph table's shared health classification treats both native `HEALTH_ERR`
+and the server-normalized `ERR` as critical. Row status, status-filter counts,
+and the drawer header use that same classification, including when no manager
+daemons are reported. Missing service rows must not turn explicit critical
+provider health into a healthy table verdict.
The Mail Gateway row reflects its open alerts, not only the provider status:
a reachable gateway with an unacknowledged alert (mail queued past its age
threshold, a backlog) renders and filters as needing attention, colours the
diff --git a/frontend-modern/browser-verification/64939644a0099e9e.json b/frontend-modern/browser-verification/64939644a0099e9e.json
new file mode 100644
index 0000000000..d3def3edf6
--- /dev/null
+++ b/frontend-modern/browser-verification/64939644a0099e9e.json
@@ -0,0 +1,55 @@
+{
+ "version": 1,
+ "base_sha": "2d380af6580ebe4f8edaeda4f3894c13af880321",
+ "verified_at": "2026-11-02T08:41:33Z",
+ "result": "passed",
+ "changed_paths": [
+ "frontend-modern/src/features/proxmox/ProxmoxCephTable.tsx"
+ ],
+ "content_sha256": {
+ "frontend-modern/src/features/proxmox/ProxmoxCephTable.tsx": "d734206f4c79ecd9ece4b8c68812d4ab5a4813c776e1fbf560a5c09d1d1dc4ab"
+ },
+ "routes": [
+ "/proxmox/ceph"
+ ],
+ "viewports": [
+ {
+ "width": 1920,
+ "height": 1080
+ },
+ {
+ "width": 1440,
+ "height": 1000
+ },
+ {
+ "width": 390,
+ "height": 844
+ }
+ ],
+ "states": [
+ "Legacy agent available manager row normalizes from 1 of 3 to 3 of 3 without a false shortage",
+ "Current agent available manager row remains 3 of 3",
+ "Starting map with an active-name hint but unavailable manager retains 0 of 3",
+ "Unavailable manager retains 0 of 2 and the native warning",
+ "No managers reported retains the native ERR critical row, filter count and drawer header",
+ "Real MON and OSD shortages remain visible while manager standbys are healthy",
+ "First healthy report clears old health details while the incident awaits confirmation",
+ "Second healthy report clears the active incident and all service shortages",
+ "Critical filter selected and unselected with pointer and keyboard focus visible",
+ "Narrow inline Filters area expanded and collapsed",
+ "Details disclosure open and closed with full FSID visible",
+ "Overview and Manage tabs selected",
+ "Drawer closed, reopened and retained across reload"
+ ],
+ "interactions": [
+ "Post real authenticated agent reports and read the published canonical resources",
+ "Hover desktop health cells and inspect the native health title",
+ "Select Critical with pointer and Space, restore All with Enter and pointer",
+ "Expand and collapse narrow Filters while retaining the selected cluster drawer",
+ "Toggle Details with Enter and inspect cluster, native health and topology",
+ "Switch Manage and Overview tabs",
+ "Close drawer with Enter, verify focus returns to the row disclosure, reopen with Space",
+ "Reload the route and verify stable selection, healthy manager count and recovered state",
+ "Inspect actual desktop, intermediate and narrow pixels for placement, clipping and focus"
+ ]
+}
diff --git a/frontend-modern/src/features/proxmox/ProxmoxCephTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxCephTable.tsx
index 7c76d35736..5f3355f3a4 100644
--- a/frontend-modern/src/features/proxmox/ProxmoxCephTable.tsx
+++ b/frontend-modern/src/features/proxmox/ProxmoxCephTable.tsx
@@ -116,7 +116,13 @@ const STATUS_FILTER_OPTIONS: PlatformTableFilterOption<CephStatusFilter>[] = [
function classify(resource: Resource): CephStatusFilter {
const raw = (resource.ceph?.healthStatus ?? resource.status ?? '').toUpperCase();
if (raw === 'HEALTH_OK' || raw === 'OK' || raw === 'ONLINE') return 'healthy';
- if (raw === 'HEALTH_ERR' || raw === 'ERROR' || raw === 'CRITICAL' || raw === 'OFFLINE') {
+ if (
+ raw === 'HEALTH_ERR' ||
+ raw === 'ERR' ||
+ raw === 'ERROR' ||
+ raw === 'CRITICAL' ||
+ raw === 'OFFLINE'
+ ) {
return 'critical';
}
if (raw === 'HEALTH_WARN' || raw === 'WARN' || raw === 'WARNING' || raw === 'DEGRADED') {
diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxCephTable.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxCephTable.test.tsx
index 71443e2ab9..e14cd728d5 100644
--- a/frontend-modern/src/features/proxmox/__tests__/ProxmoxCephTable.test.tsx
+++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxCephTable.test.tsx
@@ -188,6 +188,37 @@ describe('ProxmoxCephTable', () => {
);
});
+ it.each(['ERR', 'HEALTH_ERR'])(
+ 'keeps %s critical when no managers are reported',
+ async (healthStatus) => {
+ const cluster = makeCluster('ceph-critical');
+ cluster.ceph = {
+ ...cluster.ceph!,
+ healthStatus,
+ services: [{ type: 'mgr', running: 0, total: 0 }],
+ };
+ render(() => (
+ <ProxmoxCephTable
+ resources={[cluster]}
+ emptyIcon={<span />}
+ emptyTitle="No Ceph clusters"
+ emptyDescription="No clusters"
+ />
+ ));
+
+ expect(screen.getByRole('cell', { name: 'Critical' })).toBeInTheDocument();
+ const heading = screen.getByRole('heading', { level: 2, name: 'ceph-critical' });
+ expect(heading.parentElement?.querySelector('[aria-label="Critical"]')).not.toBeNull();
+ expect(screen.getByTestId('ceph-cluster-services')).toHaveTextContent('None reported');
+ const critical = screen.getByRole('button', { name: 'Critical, 1' });
+ await fireEvent.click(critical);
+ expect(critical).toHaveAttribute('aria-pressed', 'true');
+ expect(screen.getByRole('cell', { name: 'Critical' })).toBeInTheDocument();
+ await fireEvent.click(screen.getByRole('button', { name: 'All, 1' }));
+ expect(critical).toHaveAttribute('aria-pressed', 'false');
+ },
+ );
+
it("heads the drawer with the row's own health verdict", () => {
// A cluster without a reported health status reads healthy in its row, so
// the drawer header must not show a grey "Unknown" beside it.
diff --git a/internal/hostagent/agent_metrics_test.go b/internal/hostagent/agent_metrics_test.go
index de4c1be566..5a6b395301 100644
--- a/internal/hostagent/agent_metrics_test.go
+++ b/internal/hostagent/agent_metrics_test.go
@@ -339,6 +339,55 @@ func TestBuildReport(t *testing.T) {
mc.cephStatusFn = nil
})
+ // One active and two standby managers, in the Octopus+ `ceph status`
+ // summary and in the full mgr map of older releases. Standby managers are
+ // running daemons; only the absence of an available manager leaves the
+ // report's mgr row at zero running.
+ t.Run("Ceph standby managers count as running", func(t *testing.T) {
+ previousGOOS := mc.goos
+ t.Cleanup(func() {
+ mc.cephStatusFn = nil
+ mc.goos = previousGOOS
+ })
+ cases := []struct {
+ name string
+ mgrmap string
+ running, total int
+ }{
+ {name: "status summary", mgrmap: `{"available":true,"num_standbys":2,"modules":[],"services":{}}`, running: 3, total: 3},
+ {name: "full mgr map", mgrmap: `{"available":true,"active_name":"pve1","standbys":[{"name":"pve2"},{"name":"pve3"}]}`, running: 3, total: 3},
+ {name: "no available manager", mgrmap: `{"available":false,"num_standbys":2,"modules":[],"services":{}}`, running: 0, total: 2},
+ }
+ for _, tc := range cases {
+ status, err := parseCephStatus([]byte(`{"fsid":"ceph-fsid-mgr","health":{"status":"HEALTH_OK"},"mgrmap":` + tc.mgrmap + `}`))
+ if err != nil {
+ t.Fatalf("%s: parseCephStatus: %v", tc.name, err)
+ }
+ mc.cephStatusFn = func(ctx context.Context) (*CephClusterStatus, error) {
+ return status, nil
+ }
+ mc.goos = "linux"
+
+ report, err := agent.buildReport(context.Background())
+ if err != nil {
+ t.Fatalf("%s: buildReport failed: %v", tc.name, err)
+ }
+ if report.Ceph == nil {
+ t.Fatalf("%s: Ceph report is nil", tc.name)
+ }
+ var mgr *agentshost.CephService
+ for i := range report.Ceph.Services {
+ if report.Ceph.Services[i].Type == "mgr" {
+ mgr = &report.Ceph.Services[i]
+ }
+ }
+ if mgr == nil || mgr.Running != tc.running || mgr.Total != tc.total {
+ t.Errorf("%s: mgr service = %+v, want %d of %d running", tc.name, mgr, tc.running, tc.total)
+ }
+ }
+ mc.cephStatusFn = nil
+ })
+
// Test case 5: Unraid collection
t.Run("Unraid collection", func(t *testing.T) {
mc.unraidStorageFn = func(ctx context.Context) (*agentshost.UnraidStorage, error) {
diff --git a/internal/hostagent/ceph.go b/internal/hostagent/ceph.go
index 28a19ed411..8839088586 100644
--- a/internal/hostagent/ceph.go
+++ b/internal/hostagent/ceph.go
@@ -367,9 +367,17 @@ func parseCephStatus(data []byte) (*CephClusterStatus, error) {
if monsRunning == 0 {
monsRunning = numMons
}
+ // A standby manager is a running ceph-mgr: every manager beacons the
+ // monitors, which drop a standby once its beacons stop, so all listed
+ // managers are running while one of them is available. Without an
+ // available manager none is serving.
+ mgrsRunning := 0
+ if raw.MgrMap.Available {
+ mgrsRunning = status.MgrMap.NumMgrs
+ }
status.Services = []CephServiceInfo{
{Type: "mon", Running: monsRunning, Total: numMons},
- {Type: "mgr", Running: cephBoolToInt(raw.MgrMap.Available), Total: status.MgrMap.NumMgrs},
+ {Type: "mgr", Running: mgrsRunning, Total: status.MgrMap.NumMgrs},
{Type: "osd", Running: raw.OSDMap.NumUp, Total: raw.OSDMap.NumOSDs},
}
@@ -415,13 +423,6 @@ func parseCephDF(data []byte) ([]CephPool, float64, error) {
return pools, normalizePercentUsed(raw.Stats.PercentUsed), nil
}
-func cephBoolToInt(b bool) int {
- if b {
- return 1
- }
- return 0
-}
-
// normalizePercentUsed accepts Ceph values reported either as ratio (0-1) or percent (0-100).
func normalizePercentUsed(value float64) float64 {
if value >= 0 && value <= 1 {
diff --git a/internal/hostagent/ceph_test.go b/internal/hostagent/ceph_test.go
index c595a73388..a744bd3aad 100644
--- a/internal/hostagent/ceph_test.go
+++ b/internal/hostagent/ceph_test.go
@@ -467,12 +467,3 @@ func TestCollectCeph_UsagePercentFromDF(t *testing.T) {
t.Fatalf("expected collected timestamp set")
}
}
-
-func TestCephBoolToInt(t *testing.T) {
- if cephBoolToInt(true) != 1 {
- t.Fatalf("expected cephBoolToInt(true)=1")
- }
- if cephBoolToInt(false) != 0 {
- t.Fatalf("expected cephBoolToInt(false)=0")
- }
-}
diff --git a/internal/hostagent/issue1626_ceph_squid_status_test.go b/internal/hostagent/issue1626_ceph_squid_status_test.go
index 47ccef9c12..83e7d8986c 100644
--- a/internal/hostagent/issue1626_ceph_squid_status_test.go
+++ b/internal/hostagent/issue1626_ceph_squid_status_test.go
@@ -68,8 +68,9 @@ func TestIssue1626ParseCephStatusSquidSchema(t *testing.T) {
if monService == nil || monService.Running != 3 || monService.Total != 3 {
t.Errorf("mon service = %+v, want Running 3 / Total 3", monService)
}
- if mgrService == nil || mgrService.Running != 1 || mgrService.Total != 2 {
- t.Errorf("mgr service = %+v, want Running 1 / Total 2", mgrService)
+ // The standby is a running ceph-mgr, not a stopped one.
+ if mgrService == nil || mgrService.Running != 2 || mgrService.Total != 2 {
+ t.Errorf("mgr service = %+v, want Running 2 / Total 2", mgrService)
}
}
diff --git a/internal/models/ceph_cluster_identity.go b/internal/models/ceph_cluster_identity.go
index b6e027f89f..48b5171c85 100644
--- a/internal/models/ceph_cluster_identity.go
+++ b/internal/models/ceph_cluster_identity.go
@@ -106,14 +106,18 @@ func supplementCephCluster(primary, supplemental CephCluster) CephCluster {
aliases = append(aliases, supplemental.InstanceAliases...)
primary.InstanceAliases = normalizeCephInstanceAliases(primary.Instance, aliases)
- if primary.Health == "" {
+ // Status and its details belong to one health observation. An explicit
+ // status owns empty details too: a recovered report must not inherit an
+ // old incident from an earlier or lower-priority source. Only an absent
+ // status permits the existing last-known health fallback.
+ if strings.TrimSpace(primary.Health) == "" {
primary.Health = supplemental.Health
- }
- if primary.HealthMessage == "" {
- primary.HealthMessage = supplemental.HealthMessage
- }
- if len(primary.HealthChecks) == 0 && len(supplemental.HealthChecks) > 0 {
- primary.HealthChecks = append([]CephHealthCheck(nil), supplemental.HealthChecks...)
+ if primary.HealthMessage == "" {
+ primary.HealthMessage = supplemental.HealthMessage
+ }
+ if len(primary.HealthChecks) == 0 && len(supplemental.HealthChecks) > 0 {
+ primary.HealthChecks = append([]CephHealthCheck(nil), supplemental.HealthChecks...)
+ }
}
if primary.TotalBytes == 0 && supplemental.TotalBytes != 0 {
primary.TotalBytes = supplemental.TotalBytes
diff --git a/internal/models/state_host_test.go b/internal/models/state_host_test.go
index 8e1faaab4f..426834f9cd 100644
--- a/internal/models/state_host_test.go
+++ b/internal/models/state_host_test.go
@@ -720,6 +720,67 @@ func TestCephClusterStatePrefersProxmoxAPIByFSID(t *testing.T) {
}
}
+// A reported status and its health details are one observation. In particular,
+// healthy reports cannot borrow a previous incident's message or checks.
+func TestCephClusterHealthDetailsFollowAuthoritativeStatus(t *testing.T) {
+ warning := func(source, instance string) CephCluster {
+ return CephCluster{ID: instance + "-fsid", Instance: instance, Source: source, FSID: "fsid-health",
+ Health: "WARN", HealthMessage: "No active Ceph manager",
+ HealthChecks: []CephHealthCheck{{Code: "MGR_DOWN", Severity: "HEALTH_WARN", Summary: "No active Ceph manager"}}, NumMgrs: 3}
+ }
+ agentWarning := warning(CephClusterSourceHostAgent, "agent")
+ apiWarning := warning(CephClusterSourceProxmoxAPI, "api")
+ agentHealthy := CephCluster{ID: "agent-fsid", Instance: "agent", Source: CephClusterSourceHostAgent, FSID: "fsid-health", Health: "OK"}
+ apiHealthy := CephCluster{ID: "api-fsid", Instance: "api", Source: CephClusterSourceProxmoxAPI, FSID: "fsid-health", Health: "HEALTH_OK"}
+ apiUnknown := apiHealthy
+ apiUnknown.Health = ""
+ apiBlank := apiUnknown
+ apiBlank.Health = " "
+ agentNewWarning := agentHealthy
+ agentNewWarning.Health = "WARN"
+ cases := []struct {
+ name string
+ reports []CephCluster
+ wantStatus string
+ wantFallback bool
+ }{
+ {"agent recovery clears the previous incident", []CephCluster{agentWarning, agentHealthy}, "OK", false},
+ {"API recovery clears the previous incident", []CephCluster{apiWarning, apiHealthy}, "HEALTH_OK", false},
+ {"API health takes authority over agent warning", []CephCluster{agentWarning, apiHealthy}, "HEALTH_OK", false},
+ {"agent warning cannot fill explicit API health", []CephCluster{apiHealthy, agentWarning}, "HEALTH_OK", false},
+ {"agent health cannot clear authoritative API warning", []CephCluster{apiWarning, agentHealthy}, "WARN", true},
+ {"missing API status retains health fallback", []CephCluster{agentWarning, apiUnknown}, "WARN", true},
+ {"blank API status retains health fallback", []CephCluster{agentWarning, apiBlank}, "WARN", true},
+ {"new warning without detail cannot inherit an old cause", []CephCluster{agentWarning, agentNewWarning}, "WARN", false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ state := NewState()
+ for _, report := range tc.reports {
+ state.UpsertCephCluster(report)
+ }
+ clusters := state.GetSnapshot().CephClusters
+ if len(clusters) != 1 {
+ t.Fatalf("clusters = %#v, want one FSID", clusters)
+ }
+ got := clusters[0]
+ if got.Health != tc.wantStatus {
+ t.Fatalf("health = %q, want %q", got.Health, tc.wantStatus)
+ }
+ if tc.wantFallback {
+ if got.HealthMessage != "No active Ceph manager" || len(got.HealthChecks) != 1 || got.HealthChecks[0].Code != "MGR_DOWN" {
+ t.Fatalf("health fallback lost: %#v", got)
+ }
+ } else if got.HealthMessage != "" || len(got.HealthChecks) != 0 {
+ t.Fatalf("explicit health inherited stale details: %#v", got)
+ }
+ if got.NumMgrs != 3 {
+ t.Fatalf("unrelated count fallback changed: managers = %d", got.NumMgrs)
+ }
+ })
+ }
+}
+
func TestUpsertCephClusterKeepsProxmoxAPIWhenAgentRefreshArrives(t *testing.T) {
state := NewState()
diff --git a/internal/monitoring/monitor_helpers.go b/internal/monitoring/monitor_helpers.go
index 8e4b37008a..89c6e7a4f0 100644
--- a/internal/monitoring/monitor_helpers.go
+++ b/internal/monitoring/monitor_helpers.go
@@ -933,7 +933,7 @@ func convertAgentCephToModels(ceph *agentshost.CephCluster) *models.HostCephClus
for _, svc := range ceph.Services {
result.Services = append(result.Services, models.HostCephService{
Type: svc.Type,
- Running: svc.Running,
+ Running: agentCephServiceRunning(ceph, svc),
Total: svc.Total,
Daemons: svc.Daemons,
})
@@ -942,6 +942,19 @@ func convertAgentCephToModels(ceph *agentshost.CephCluster) *models.HostCephClus
return result
}
+// agentCephServiceRunning returns the running count for an agent service row.
+// Agents built before this fix counted only the active manager as running, so
+// a healthy cluster with two standby managers read "1 of 3 running". Standby
+// managers are running daemons that the monitors drop once their beacons stop,
+// so while a manager is available every listed manager counts as running,
+// whichever agent version sent the row.
+func agentCephServiceRunning(ceph *agentshost.CephCluster, svc agentshost.CephService) int {
+ if ceph.MgrMap.Available && strings.EqualFold(strings.TrimSpace(svc.Type), "mgr") {
+ return max(svc.Running, svc.Total)
+ }
+ return svc.Running
+}
+
// convertAgentCephToGlobalCluster converts agent Ceph data to the global CephCluster format
// used by the State.CephClusters list.
func convertAgentCephToGlobalCluster(ceph *agentshost.CephCluster, hostname, hostID string, timestamp time.Time) models.CephCluster {
@@ -1009,7 +1022,7 @@ func convertAgentCephToGlobalCluster(ceph *agentshost.CephCluster, hostname, hos
for _, svc := range ceph.Services {
cluster.Services = append(cluster.Services, models.CephServiceStatus{
Type: svc.Type,
- Running: svc.Running,
+ Running: agentCephServiceRunning(ceph, svc),
Total: svc.Total,
})
}
diff --git a/internal/monitoring/monitor_host_agents_test.go b/internal/monitoring/monitor_host_agents_test.go
index 156ce6b283..e79ad90d42 100644
--- a/internal/monitoring/monitor_host_agents_test.go
+++ b/internal/monitoring/monitor_host_agents_test.go
@@ -5148,6 +5148,75 @@ func TestApplyHostReportMergesAgentCephWithProxmoxAPICluster(t *testing.T) {
t.Fatalf("expected canonical API Ceph pool alert using legacy agent override, got active alerts: %#v", active)
}
+// Agents before the manager fix sent the Ceph mgr row as "1 of N running"
+// whenever a manager was available, counting standby managers as stopped.
+// Standbys are running ceph-mgr daemons, so ingest must publish every listed
+// manager as running while one is available, whichever agent sent the row, and
+// keep zero running when none is available. Other rows pass through unchanged.
+func TestApplyHostReportCountsStandbyCephManagersAsRunning(t *testing.T) {
+ cases := []struct {
+ name string
+ available bool
+ mgr agentshost.CephService
+ want int
+ }{
+ {name: "older agent with an available manager", available: true, mgr: agentshost.CephService{Type: "mgr", Running: 1, Total: 3}, want: 3},
+ {name: "fixed agent with an available manager", available: true, mgr: agentshost.CephService{Type: "mgr", Running: 3, Total: 3}, want: 3},
+ {name: "no available manager", available: false, mgr: agentshost.CephService{Type: "mgr", Running: 0, Total: 2}, want: 0},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ monitor := &Monitor{
+ state: models.NewState(),
+ alertManager: alerts.NewManager(),
+ hostTokenBindings: make(map[string]string),
+ config: &config.Config{},
+ rateTracker: NewRateTracker(),
+ }
+ t.Cleanup(func() { monitor.alertManager.Stop() })
+
+ report := agentshost.Report{
+ Agent: agentshost.AgentInfo{ID: "agent-ceph-mgr", Version: "1.0.0", IntervalSeconds: 30},
+ Host: agentshost.HostInfo{ID: "pve1-host", Hostname: "pve1", Platform: "linux"},
+ Timestamp: time.Now().UTC(),
+ Ceph: &agentshost.CephCluster{
+ FSID: "ceph-fsid-mgr",
+ Health: agentshost.CephHealth{Status: "HEALTH_OK"},
+ MgrMap: agentshost.CephManagerMap{Available: tc.available, NumMgrs: tc.mgr.Total, Standbys: 2},
+ Services: []agentshost.CephService{
+ {Type: "mon", Running: 2, Total: 3},
+ tc.mgr,
+ {Type: "osd", Running: 5, Total: 6},
+ },
+ },
+ }
+ if _, err := monitor.ApplyHostReport(report, nil); err != nil {
+ t.Fatalf("ApplyHostReport: %v", err)
+ }
+
+ want := []models.CephServiceStatus{
+ {Type: "mon", Running: 2, Total: 3},
+ {Type: "mgr", Running: tc.want, Total: tc.mgr.Total},
+ {Type: "osd", Running: 5, Total: 6},
+ }
+ snapshot := monitor.state.GetSnapshot()
+ if len(snapshot.CephClusters) != 1 || !reflect.DeepEqual(snapshot.CephClusters[0].Services, want) {
+ t.Fatalf("Ceph cluster services = %+v, want %+v", snapshot.CephClusters, want)
+ }
+ if len(snapshot.Hosts) != 1 || snapshot.Hosts[0].Ceph == nil {
+ t.Fatalf("expected one host with Ceph data, got %+v", snapshot.Hosts)
+ }
+ hostRows := make([]models.CephServiceStatus, 0, len(snapshot.Hosts[0].Ceph.Services))
+ for _, svc := range snapshot.Hosts[0].Ceph.Services {
+ hostRows = append(hostRows, models.CephServiceStatus{Type: svc.Type, Running: svc.Running, Total: svc.Total})
+ }
+ if !reflect.DeepEqual(hostRows, want) {
+ t.Fatalf("host Ceph services = %+v, want %+v", hostRows, want)
+ }
+ })
+ }
+}
+
func TestApplyHostReportMapsReclaimableMemoryCache(t *testing.T) {
monitor := &Monitor{
state: models.NewState(),