Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Pin Podman CPU selection through reports, History and alerts

pulse-triage[bot]e3917eee6bc1
diff --git a/internal/dockeragent/cpu_roundtrip_test.go b/internal/dockeragent/cpu_roundtrip_test.go
new file mode 100644
index 0000000000..9732e240e9
--- /dev/null
+++ b/internal/dockeragent/cpu_roundtrip_test.go
@@ -0,0 +1,224 @@
+package dockeragent
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "math"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ containertypes "github.com/moby/moby/api/types/container"
+ systemtypes "github.com/moby/moby/api/types/system"
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+ "github.com/rcourtman/pulse-go-rewrite/internal/api"
+ "github.com/rcourtman/pulse-go-rewrite/internal/config"
+ "github.com/rcourtman/pulse-go-rewrite/internal/hostmetrics"
+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
+ agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
+ "github.com/rs/zerolog"
+)
+
+// The deliberately conflicting compatibility percentages below are synthetic,
+// not captured Podman responses. Walk the actual collector, compressed HTTPS
+// report, authenticated ingestion, memory/store/HTTP History and alert manager.
+// This reproduces a source mechanism, not the reporter's installed cause.
+func TestPodmanCPUCollectorHistoryAndAlerts(t *testing.T) {
+ t.Setenv("PULSE_DEV", "true")
+ swap(t, &hostmetricsCollect, func(context.Context, []string) (hostmetrics.Snapshot, error) {
+ return hostmetrics.Snapshot{}, nil
+ })
+ for _, cpus := range []int{8, 4} {
+ t.Run(fmt.Sprintf("%d-cpu", cpus), func(t *testing.T) {
+ const reportToken = "podman-cpu-report-fixture.12345678"
+ const readToken = "podman-cpu-read-fixture.12345678"
+ reporter, err := config.NewAPITokenRecord(reportToken, "CPU report fixture", []string{config.ScopeDockerReport})
+ if err != nil {
+ t.Fatal(err)
+ }
+ reader, err := config.NewAPITokenRecord(readToken, "CPU History fixture", []string{config.ScopeMonitoringRead})
+ if err != nil {
+ t.Fatal(err)
+ }
+ dir := t.TempDir()
+ cfg := &config.Config{DataPath: dir, ConfigPath: dir, APITokens: []config.APITokenRecord{*reporter, *reader}}
+ monitor, err := monitoring.New(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(monitor.Stop)
+ if monitor.GetMetricsStore() == nil {
+ t.Fatal("persistent History store unavailable")
+ }
+ router := api.NewRouter(cfg, monitor, nil, nil, nil, "test")
+ t.Cleanup(router.ShutdownResourceStores)
+ t.Cleanup(router.ShutdownRBAC)
+ t.Cleanup(router.ShutdownBackgroundWorkers)
+ server := httptest.NewTLSServer(router.Handler())
+ t.Cleanup(server.Close)
+
+ alertConfig := monitor.GetAlertManager().GetConfig()
+ alertConfig.Enabled = true
+ alertConfig.ActivationState = alerts.ActivationActive
+ alertConfig.Schedule.QuietHours.Enabled = false
+ alertConfig.DockerDefaults.CPU = alerts.HysteresisThreshold{Trigger: 80, Clear: 70}
+ alertConfig.MetricTimeThresholds = map[string]map[string]int{"all": {"cpu": 0}}
+ monitor.GetAlertManager().UpdateConfig(alertConfig)
+
+ epoch := time.Now().UTC().Add(-time.Minute)
+ type sample struct {
+ total, halfTotal uint64
+ native, halfNative float64
+ want, halfWant float64
+ start int
+ }
+ // The first three disagreements can produce the reported 74.3/88.0/
+ // 90.7 capacity values on 8 CPUs with the old positive-value override.
+ samples := []sample{
+ {total: 1_000_000_000, halfTotal: 500_000_000},
+ {total: 1_999_900_000, halfTotal: 1_000_400_000, native: 594.4, halfNative: 725.6, want: 99.99, halfWant: 50.04},
+ {total: 2_999_700_000, halfTotal: 1_500_400_000, native: 704, halfNative: 92.8, want: 99.98, halfWant: 50},
+ {total: 2_999_700_000, halfTotal: 1_500_400_000, native: 800, halfNative: 148.8}, // Idle, not a positive-value fallback.
+ {total: 10_000_000, halfTotal: 5_000_000, native: 800, halfNative: 800}, // Counter reset.
+ {total: 10_000_000 + uint64(cpus)*900_000_000, halfTotal: 505_000_000, native: 1, halfNative: 1, want: float64(cpus) * 90, halfWant: 50}, // Real sustained high usage still alerts.
+ {total: 20_000_000_000, halfTotal: 20_000_000_000, native: 800, halfNative: 800, start: 1}, // Restart with a larger lifetime counter.
+ }
+ var current sample
+ var index int
+ fullID, halfID := "podman-full-core-123456", "podman-half-core-123456"
+ agent := &Agent{
+ cfg: Config{AgentID: fmt.Sprintf("podman-cpu-host-%d", cpus), IncludeContainers: true, Interval: time.Second},
+ runtime: RuntimePodman, logger: zerolog.Nop(), agentVersion: "test", reportStreamID: "cpu-roundtrip-stream",
+ targets: []TargetConfig{{URL: server.URL, Token: reportToken}},
+ httpClients: map[bool]*http.Client{false: server.Client()},
+ docker: &fakeDockerClient{
+ infoFunc: func(context.Context) (systemtypes.Info, error) {
+ return systemtypes.Info{Name: "cpu-host", NCPU: cpus, ServerVersion: "5.4.2"}, nil
+ },
+ containerListFunc: func(context.Context, dockerContainerListOptions) ([]containertypes.Summary, error) {
+ return []containertypes.Summary{
+ {ID: fullID, Names: []string{"/full-core"}, State: "running", Status: "Up"},
+ {ID: halfID, Names: []string{"/half-core"}, State: "running", Status: "Up"},
+ }, nil
+ },
+ containerInspectWithRawFn: func(context.Context, string, bool) (containertypes.InspectResponse, []byte, error) {
+ inspect := baseInspect()
+ inspect.State = &containertypes.State{Running: true, StartedAt: epoch.Add(time.Duration(current.start-60) * time.Second).Format(time.RFC3339Nano)}
+ return inspect, nil, nil
+ },
+ containerStatsOneShotFn: func(_ context.Context, id string) (dockerStatsResponseReader, error) {
+ total, percent := current.total, current.native
+ if id == halfID {
+ total, percent = current.halfTotal, current.halfNative
+ }
+ payload := fmt.Sprintf(`{"read":%q,"cpu_stats":{"cpu_usage":{"total_usage":%d},"system_cpu_usage":%d,"online_cpus":%d,"cpu":%g},"memory_stats":{"usage":1000000,"limit":4000000}}`,
+ epoch.Add(time.Duration(index)*time.Second).Format(time.RFC3339Nano), total, uint64(index+1)*1_000_000_000, cpus, percent)
+ return dockerStatsResponseReader{Body: io.NopCloser(strings.NewReader(payload))}, nil
+ },
+ },
+ }
+ requestHistory := func(token, id string) *httptest.ResponseRecorder {
+ req := httptest.NewRequest(http.MethodGet, "/api/metrics-store/history?resourceType=app-container&resourceId="+id+"&metric=cpu&range=5m", nil)
+ if token != "" {
+ req.Header.Set("X-API-Token", token)
+ }
+ rec := httptest.NewRecorder()
+ router.Handler().ServeHTTP(rec, req)
+ return rec
+ }
+ for _, test := range []struct {
+ token string
+ status int
+ }{{"", http.StatusUnauthorized}, {reportToken, http.StatusForbidden}} {
+ if rec := requestHistory(test.token, fullID); rec.Code != test.status {
+ t.Fatalf("History auth/scope status %d, want %d", rec.Code, test.status)
+ }
+ }
+
+ var lastReport agentsdocker.Report
+ for i, s := range samples {
+ if i > 0 {
+ // The persistent raw tier has whole-second timestamps. Give each
+ // real ingestion a distinct bucket rather than inserting fake rows.
+ time.Sleep(time.Until(time.Now().Truncate(time.Second).Add(time.Second)) + 10*time.Millisecond)
+ }
+ index, current = i, s
+ report, err := agent.buildReport(context.Background())
+ if err != nil || len(report.Containers) != 2 {
+ t.Fatalf("build sample %d: %v, containers %d", i, err, len(report.Containers))
+ }
+ before := time.Now()
+ if err := agent.sendReport(context.Background(), report); err != nil {
+ t.Fatalf("send sample %d: %v", i, err)
+ }
+ after := time.Now()
+ lastReport = report
+ host, ok := monitor.GetDockerHost(agent.cfg.AgentID)
+ if !ok || host.CPUs != cpus || host.Runtime != "podman" || len(host.Containers) != 2 {
+ t.Fatalf("ingested identity/capacity changed: %+v", host)
+ }
+ monitor.GetMetricsStore().Flush()
+ for _, c := range host.Containers {
+ want := s.want
+ if c.ID == halfID {
+ want = s.halfWant
+ }
+ capacity := want / float64(cpus)
+ if math.Abs(c.CPUPercent-want) > 1e-8 || math.Abs(c.CPUCapacityPercent-capacity) > 1e-8 || c.MemoryUsage != 1_000_000 || c.MemoryPercent != 25 {
+ t.Errorf("sample %d container %s: raw %g capacity %g memory %g, want raw %g capacity %g memory 25", i, c.ID, c.CPUPercent, c.CPUCapacityPercent, c.MemoryPercent, want, capacity)
+ }
+ memory := monitor.GetMetricsHistory().GetGuestMetrics("docker:"+c.ID, "cpu", 5*time.Minute)
+ if len(memory) != i+1 || math.Abs(memory[len(memory)-1].Value-capacity) > 1e-8 || memory[len(memory)-1].Timestamp.Before(before) || memory[len(memory)-1].Timestamp.After(after) {
+ t.Errorf("sample %d memory History for %s = %+v, want capacity %g at receipt time", i, c.ID, memory, capacity)
+ }
+ rec := requestHistory(readToken, c.ID)
+ var history struct {
+ ResourceType string `json:"resourceType"`
+ ResourceID string `json:"resourceId"`
+ Metric string `json:"metric"`
+ Source string `json:"source"`
+ Points []struct {
+ Timestamp int64 `json:"timestamp"`
+ Value float64 `json:"value"`
+ } `json:"points"`
+ }
+ if rec.Code != http.StatusOK || json.Unmarshal(rec.Body.Bytes(), &history) != nil || history.ResourceType != "app-container" || history.ResourceID != c.ID || history.Metric != "cpu" || history.Source != "store" || len(history.Points) != i+1 {
+ t.Fatalf("sample %d stored HTTP History identity/coverage: %d %s", i, rec.Code, rec.Body.String())
+ }
+ point := history.Points[len(history.Points)-1]
+ if math.Abs(point.Value-capacity) > 1e-8 || point.Timestamp < before.Truncate(time.Second).UnixMilli() || point.Timestamp > after.Truncate(time.Second).UnixMilli() {
+ t.Errorf("sample %d stored HTTP History %s: %+v, want capacity %g at receipt time", i, c.ID, point, capacity)
+ }
+ }
+ active := monitor.GetAlertManager().GetActiveAlerts()
+ if i == 5 {
+ if len(active) != 1 || active[0].Value != 90 || active[0].Metadata["containerId"] != fullID || active[0].Metadata["cpuRawPercent"] != float64(cpus)*90 || active[0].Metadata["cpuCapacityCores"] != cpus {
+ t.Errorf("real high usage did not reach the correct alert identity/units: %+v", active)
+ }
+ } else if len(active) != 0 {
+ t.Errorf("sample %d conflicting one-shot percentage manufactured alerts: %+v", i, active)
+ }
+ t.Logf("sample %d: checked final compressed report, two raw/capacity identities, memory/store HTTP History and %d active alerts on %d CPUs", i, len(active), cpus)
+ }
+ // A duplicated final report must not refresh or add History samples.
+ if err := agent.sendReport(context.Background(), lastReport); err != nil {
+ t.Fatal(err)
+ }
+ if n := len(monitor.GetMetricsHistory().GetGuestMetrics("docker:"+fullID, "cpu", 5*time.Minute)); n != len(samples) {
+ t.Fatalf("duplicate report added History: %d points", n)
+ }
+ // The History credential must not admit report writes.
+ req := httptest.NewRequest(http.MethodPost, "/api/agents/docker/report", strings.NewReader(`{}`))
+ req.Header.Set("X-API-Token", readToken)
+ rec := httptest.NewRecorder()
+ router.Handler().ServeHTTP(rec, req)
+ if rec.Code != http.StatusForbidden {
+ t.Fatalf("read token admitted report write: %d", rec.Code)
+ }
+ })
+ }
+}