Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Count API Ceph standby managers as running
pulse-triage[bot]d888d384d228
diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.mdindex 57cb0a6aa3..3057cdde2a 100644--- a/docs/release-control/v6/internal/subsystems/monitoring.md+++ b/docs/release-control/v6/internal/subsystems/monitoring.md@@ -6139,6 +6139,15 @@ reports cannot inherit an older or lower-priority incident. Only an absentstatus permits the existing health continuity fallback. Other topology/countfallbacks and source priority remain independent. `internal/models/state_host_test.go`checks recovery, source authority and missing-status continuity.+The Proxmox API service-map projection in `internal/monitoring/ceph.go` also+counts `standby` as running for `mgr` daemons, case-insensitively. This role+does not imply healthy MON/OSD daemons and does not override explicit stopped,+down or unknown manager states from a healthy cluster summary. API source+priority and absent-service supplementation are unchanged.+`TestBuildCephClusterModelServiceRoles` and+`TestPollCephClusterMixedSourceServiceHealth` in `internal/monitoring/ceph_test.go`+pin individual roles, both API/agent arrival orders, genuine manager loss,+MON/OSD warnings and the summary-only fallback through polling and host ingest.Unraid long-running operation state uses the same authoritative transitionboundary. A non-empty active sync action may carry progress; cancellation,completion, or idle state is represented by an empty action and normalizeddiff --git a/internal/monitoring/ceph.go b/internal/monitoring/ceph.goindex 6d609721e2..65359d8b0b 100644--- a/internal/monitoring/ceph.go+++ b/internal/monitoring/ceph.go@@ -141,7 +141,11 @@ func buildCephClusterModel(instanceName string, status *proxmox.CephStatus, df *var offline []stringfor daemonName, daemon := range definition.Daemons {total++- if strings.EqualFold(daemon.Status, "running") || strings.EqualFold(daemon.Status, "active") {+ // Standby is a healthy running role for a manager, not an+ // offline daemon. Do not extend it to MON/OSD or infer health+ // for stopped/unknown daemons from the cluster summary.+ if strings.EqualFold(daemon.Status, "running") || strings.EqualFold(daemon.Status, "active") ||+ (strings.EqualFold(serviceType, "mgr") && strings.EqualFold(daemon.Status, "standby")) {running++continue}diff --git a/internal/monitoring/ceph_test.go b/internal/monitoring/ceph_test.goindex 0116ba51c6..83d8998d67 100644--- a/internal/monitoring/ceph_test.go+++ b/internal/monitoring/ceph_test.go@@ -5,14 +5,212 @@ import ("encoding/json""fmt""testing"+ "time""github.com/rcourtman/pulse-go-rewrite/internal/alerts"+ "github.com/rcourtman/pulse-go-rewrite/internal/config""github.com/rcourtman/pulse-go-rewrite/internal/models"+ agentshost "github.com/rcourtman/pulse-go-rewrite/pkg/agents/host""github.com/rcourtman/pulse-go-rewrite/pkg/proxmox""github.com/stretchr/testify/assert""github.com/stretchr/testify/mock")+func TestBuildCephClusterModelServiceRoles(t *testing.T) {+ for _, tc := range []struct {+ service, status string+ wantRunning int+ }{+ {"mgr", "active", 1},+ {"mgr", "running", 1},+ {"mgr", "standby", 1},+ {"MGR", "STANDBY", 1},+ {"mgr", "stopped", 0},+ {"mgr", "down", 0},+ {"mgr", "unknown", 0},+ {"mgr", "", 0},+ {"mon", "running", 1},+ {"mon", "standby", 0},+ {"osd", "running", 1},+ {"osd", "standby", 0},+ } {+ t.Run(tc.service+"/"+tc.status, func(t *testing.T) {+ status := &proxmox.CephStatus{+ Health: proxmox.CephHealth{Status: "HEALTH_OK"},+ ServiceMap: proxmox.CephServiceMap{Services: map[string]proxmox.CephServiceDefinition{+ tc.service: {Daemons: map[string]proxmox.CephServiceDaemon{+ "daemon": {Host: "node", Status: tc.status},+ }},+ }},+ }+ cluster := buildCephClusterModel("pve", status, nil)+ want := models.CephServiceStatus{Type: tc.service, Running: tc.wantRunning, Total: 1}+ if tc.wantRunning == 0 {+ want.Message = "Offline: daemon@node"+ }+ assert.Equal(t, []models.CephServiceStatus{want}, cluster.Services)+ })+ }+}++// Exercise the API poll and accepted host report, rather than making agent+// data canonical to hide an inconsistent API service row (issue #1626).+func TestPollCephClusterMixedSourceServiceHealth(t *testing.T) {+ for _, scenario := range []string{"healthy", "manager_down", "manager_unavailable", "mon_osd_down", "summary_only"} {+ for _, apiFirst := range []bool{true, false} {+ order := "agent_then_api"+ if apiFirst {+ order = "api_then_agent"+ }+ t.Run(scenario+"/"+order, func(t *testing.T) {+ snapshot := cephMixedSourceServiceHealthSnapshot(t, scenario, apiFirst)+ if !assert.Len(t, snapshot.CephClusters, 1) || !assert.Len(t, snapshot.Hosts, 1) {+ return+ }+ cluster := snapshot.CephClusters[0]+ assert.Equal(t, models.CephClusterSourceProxmoxAPI, cluster.Source)+ assert.Equal(t, "pve-api-ceph-mixed-services", cluster.ID)+ assert.Equal(t, 3, cluster.NumMons)+ assert.Equal(t, 2, cluster.NumMgrs)+ services := make(map[string]models.CephServiceStatus)+ for _, service := range cluster.Services {+ services[service.Type] = service+ }+ mgr := models.CephServiceStatus{Type: "mgr", Running: 2, Total: 2}+ health := "HEALTH_OK"+ switch scenario {+ case "manager_down":+ mgr.Running, mgr.Message = 1, "Offline: spare@pve-b"+ health = "HEALTH_WARN"+ case "manager_unavailable":+ mgr.Running = 0+ assert.Contains(t, []string{"Offline: active@pve-a, spare@pve-b", "Offline: spare@pve-b, active@pve-a"}, services["mgr"].Message)+ mgr.Message = services["mgr"].Message+ health = "HEALTH_ERR"+ if assert.Len(t, cluster.HealthChecks, 1) {+ assert.Equal(t, "MGR_DOWN", cluster.HealthChecks[0].Code)+ }+ case "mon_osd_down":+ health = "HEALTH_WARN"+ assert.Equal(t, models.CephServiceStatus{Type: "mon", Running: 2, Total: 3, Message: "Offline: c@pve-c"}, services["mon"])+ assert.Equal(t, models.CephServiceStatus{Type: "osd", Running: 2, Total: 3, Message: "Offline: 2@pve-c"}, services["osd"])+ assert.Equal(t, 2, cluster.NumOSDsUp)+ assert.Len(t, cluster.HealthChecks, 2)+ codes := make([]string, 0, len(cluster.HealthChecks))+ for _, check := range cluster.HealthChecks {+ codes = append(codes, check.Code)+ }+ assert.ElementsMatch(t, []string{"MON_DOWN", "OSD_DOWN"}, codes)+ assert.Contains(t, cluster.HealthMessage, "monitor down")+ assert.Contains(t, cluster.HealthMessage, "osd down")+ }+ assert.Equal(t, mgr, services["mgr"])+ assert.Equal(t, health, cluster.Health)+ if health == "HEALTH_OK" {+ assert.Empty(t, cluster.HealthChecks)+ assert.Empty(t, cluster.HealthMessage)+ }+ if assert.NotNil(t, snapshot.Hosts[0].Ceph) {+ // The intentionally healthy agent must not erase API down+ // evidence; its own machine projection remains independent.+ for _, service := range snapshot.Hosts[0].Ceph.Services {+ if service.Type == "mgr" {+ assert.Equal(t, 2, service.Running)+ assert.Equal(t, 2, service.Total)+ }+ }+ }+ })+ }+ }+}++func cephMixedSourceServiceHealthSnapshot(t *testing.T, scenario string, apiFirst bool) models.StateSnapshot {+ t.Helper()+ var status proxmox.CephStatus+ err := json.Unmarshal([]byte(`{+ "fsid":"ceph-mixed-services","health":{"status":"HEALTH_OK"},+ "monmap":{"num_mons":3},"mgrmap":{"available":true,"num_standbys":1},+ "osdmap":{"num_osds":3,"num_up_osds":3,"num_in_osds":3},+ "pgmap":{"num_pgs":32,"bytes_total":1000,"bytes_used":100,"bytes_avail":900},+ "servicemap":{"services":{+ "mon":{"daemons":{"a":{"hostname":"pve-a","status":"running"},"b":{"hostname":"pve-b","status":"running"},"c":{"hostname":"pve-c","status":"running"}}},+ "mgr":{"daemons":{"active":{"hostname":"pve-a","status":"active"},"spare":{"hostname":"pve-b","status":"standby"}}},+ "osd":{"daemons":{"0":{"hostname":"pve-a","status":"running"},"1":{"hostname":"pve-b","status":"running"},"2":{"hostname":"pve-c","status":"running"}}}+ }}+ }`), &status)+ if err != nil {+ t.Fatal(err)+ }+ setDaemonStatus := func(service, daemon, state string) {+ d := status.ServiceMap.Services[service].Daemons[daemon]+ d.Status = state+ status.ServiceMap.Services[service].Daemons[daemon] = d+ }+ switch scenario {+ case "healthy":+ case "manager_down":+ setDaemonStatus("mgr", "spare", "stopped")+ status.Health.Status = "HEALTH_WARN"+ case "manager_unavailable":+ setDaemonStatus("mgr", "active", "stopped")+ setDaemonStatus("mgr", "spare", "stopped")+ status.MgrMap = proxmox.CephMgrMap{Available: false, NumMgrs: 2}+ status.Health.Status = "HEALTH_ERR"+ status.Health.Checks = map[string]proxmox.CephHealthCheckRaw{+ "MGR_DOWN": {Severity: "HEALTH_ERR", Summary: json.RawMessage(`{"message":"no active manager"}`)},+ }+ case "mon_osd_down":+ setDaemonStatus("mon", "c", "stopped")+ setDaemonStatus("osd", "2", "down")+ status.OSDMap.NumUpOSDs = 2+ status.Health.Status = "HEALTH_WARN"+ status.Health.Checks = map[string]proxmox.CephHealthCheckRaw{+ "MON_DOWN": {Severity: "HEALTH_WARN", Summary: json.RawMessage(`{"message":"1 monitor down"}`)},+ "OSD_DOWN": {Severity: "HEALTH_WARN", Summary: json.RawMessage(`{"message":"1 osd down"}`)},+ }+ case "summary_only":+ status.ServiceMap.Services = nil+ default:+ t.Fatalf("unknown Ceph service scenario %q", scenario)+ }+ m := &Monitor{state: models.NewState(), alertManager: alerts.NewManager(), config: &config.Config{},+ hostTokenBindings: make(map[string]string), rateTracker: NewRateTracker()}+ t.Cleanup(m.alertManager.Stop)+ client := &mockCephPVEClient{}+ client.On("GetCephStatus", mock.Anything).Return(&status, nil).Once()+ client.On("GetCephDF", mock.Anything).Return(nil, nil).Once()+ t.Cleanup(func() { client.AssertExpectations(t) })+ applyAPI := func() { m.pollCephCluster(context.Background(), "pve-api", client, true) }+ applyAgent := func() {+ _, err := m.ApplyHostReport(agentshost.Report{+ Agent: agentshost.AgentInfo{ID: "mixed-agent", Version: "6.7.0", IntervalSeconds: 30},+ Host: agentshost.HostInfo{ID: "mixed-host", Hostname: "pve-a", Platform: "linux"},+ Timestamp: time.Now().UTC(),+ Ceph: &agentshost.CephCluster{+ FSID: "ceph-mixed-services", Health: agentshost.CephHealth{Status: "HEALTH_OK"},+ MonMap: agentshost.CephMonitorMap{NumMons: 3},+ MgrMap: agentshost.CephManagerMap{Available: true, NumMgrs: 2, Standbys: 1},+ OSDMap: agentshost.CephOSDMap{NumOSDs: 3, NumUp: 3, NumIn: 3},+ Services: []agentshost.CephService{+ {Type: "mon", Running: 3, Total: 3}, {Type: "mgr", Running: 1, Total: 2}, {Type: "osd", Running: 3, Total: 3},+ },+ },+ }, nil)+ if err != nil {+ t.Fatal(err)+ }+ }+ if apiFirst {+ applyAPI()+ applyAgent()+ } else {+ applyAgent()+ applyAPI()+ }+ return m.state.GetSnapshot()+}+type mockCephPVEClient struct {mock.MockPVEClientInterface