Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep requested metric history with its current reader
pulse-triage[bot]8a6bce139535
diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.mdindex e854c36eb5..9a9a192d9e 100644--- a/docs/release-control/v6/internal/subsystems/api-contracts.md+++ b/docs/release-control/v6/internal/subsystems/api-contracts.md@@ -21,6 +21,37 @@## Purpose+### Requested metric history retains its admitted reader++The existing `/api/metrics-store/history` chart read and canonical agent+`get_metric_history` read retain the admitted API token or local browser session+through the complete synchronous lookup and response. Current monitoring-read+scope, credential identity, principal and organisation authority are checked at+output, including after the agent's inventory lookup. Withdrawal before output+returns the fixed 401 rather than metric values or a successful empty series;+after an admitted prefix it only stops further writes. Handler-specific headers+remain undisclosed until an authorised write.++This uses the existing response-authority guard and changes no initial gate,+resource/metric mapping, provider work lifetime, downsampling, fallback, response+shape or retention entitlement. Removing unrelated execution/settings scopes does+not end a permitted monitoring read. Existing proxy and unauthenticated policies+are unchanged. Other readers and canonical metrics are not cancelled or deleted.+It does not erase data already admitted to an HTTP response or a client cache,+and does not establish installed session/IdP recovery or measured exposure.++Normal-suite metric-history lifetime tests hold the real resource provider after+registered-route admission, without database locks or production hooks. The+contract matrix covers both token transports, persisted HTTP token deletion,+expiry, scope/organisation/identity/principal/runtime-role withdrawal, and an+independent reader. Session tests cover actual persisted logout, replacement,+expiry, principal and session-store changes, for single and all-series responses.+Route-inventory controls retain monitoring-only, wildcard, session and proxy+reads and unrelated-scope removal. Existing history query/entitlement/fallback+and shared response-writer tests remain applicable. Consuming browser proof must+use actual producer responses at desktop and narrow sizes, not a guard skip.++### Agent HTTP redirects retain their configured originManifest discovery, capability HTTP calls and agent SSE reads using standard Godiff --git a/docs/release-control/v6/internal/subsystems/security-privacy.md b/docs/release-control/v6/internal/subsystems/security-privacy.mdindex d59a62225c..e975bfebdf 100644--- a/docs/release-control/v6/internal/subsystems/security-privacy.md+++ b/docs/release-control/v6/internal/subsystems/security-privacy.md@@ -18,6 +18,37 @@## Purpose+### Requested metric history retains its admitted reader++The existing `/api/metrics-store/history` chart read and canonical agent+`get_metric_history` read retain the admitted API token or local browser session+through the complete synchronous lookup and response. Current monitoring-read+scope, credential identity, principal and organisation authority are checked at+output, including after the agent's inventory lookup. Withdrawal before output+returns the fixed 401 rather than metric values or a successful empty series;+after an admitted prefix it only stops further writes. Handler-specific headers+remain undisclosed until an authorised write.++This uses the existing response-authority guard and changes no initial gate,+resource/metric mapping, provider work lifetime, downsampling, fallback, response+shape or retention entitlement. Removing unrelated execution/settings scopes does+not end a permitted monitoring read. Existing proxy and unauthenticated policies+are unchanged. Other readers and canonical metrics are not cancelled or deleted.+It does not erase data already admitted to an HTTP response or a client cache,+and does not establish installed session/IdP recovery or measured exposure.++Normal-suite metric-history lifetime tests hold the real resource provider after+registered-route admission, without database locks or production hooks. The+contract matrix covers both token transports, persisted HTTP token deletion,+expiry, scope/organisation/identity/principal/runtime-role withdrawal, and an+independent reader. Session tests cover actual persisted logout, replacement,+expiry, principal and session-store changes, for single and all-series responses.+Route-inventory controls retain monitoring-only, wildcard, session and proxy+reads and unrelated-scope removal. Existing history query/entitlement/fallback+and shared response-writer tests remain applicable. Consuming browser proof must+use actual producer responses at desktop and narrow sizes, not a guard skip.++### Audit viewer results cannot outlive their reader or pageAudit event details and verification outcomes in Settings are reader-owned.diff --git a/internal/api/contract_test.go b/internal/api/contract_test.goindex 6e1122df4b..35a91325bb 100644--- a/internal/api/contract_test.go+++ b/internal/api/contract_test.go@@ -29241,3 +29241,41 @@ func TestAgentHistoryWithdrawnTokenCannotDisclose(t *testing.T) {}}}+func TestMetricHistoryWithdrawnTokenCannotDisclose(t *testing.T) {+ for _, entry := range []string{"chart", "agent"} {+ for _, transport := range []string{"header", "bearer"} {+ for _, change := range []string{"revoked", "expired", "monitoring_removed", "org_removed", "credential_replaced", "principal_replaced", "invalid_role"} {+ t.Run(entry+"/"+transport+"/"+change, func(t *testing.T) {+ f := newMetricAuthorityFixture(t, entry, []string{config.ScopeMonitoringRead, config.ScopeAIExecute})+ control := f.begin(t, metricAuthorityControl, "header")+ primary := f.begin(t, metricAuthorityToken, transport)+ if change == "revoked" {+ f.revokeThroughHTTP(t)+ } else {+ config.Mu.Lock()+ record := &f.router.config.APITokens[0]+ switch change {+ case "expired":+ past := time.Now().Add(-time.Second)+ record.ExpiresAt = &past+ case "monitoring_removed":+ record.Scopes = []string{config.ScopeAIExecute}+ case "org_removed":+ record.OrgIDs = []string{"other-org"}+ case "credential_replaced":+ record.ID = "replacement-metric-credential"+ case "principal_replaced":+ record.Metadata = map[string]string{apiTokenMetadataOwnerUserID: "replacement-metric-principal"}+ case "invalid_role":+ record.Metadata = map[string]string{authpkg.RuntimeRoleMetadataKey: authpkg.RuntimeRoleActionRunner}+ }+ config.Mu.Unlock()+ }+ f.provider.unblock()+ readMetricAuthorityResult(t, primary, http.StatusUnauthorized)+ readMetricAuthorityResult(t, control, http.StatusOK)+ })+ }+ }+ }+}diff --git a/internal/api/metric_history_authority_test.go b/internal/api/metric_history_authority_test.gonew file mode 100644index 0000000000..cf1c56ad73--- /dev/null+++ b/internal/api/metric_history_authority_test.go@@ -0,0 +1,287 @@+package api++import (+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "sync"+ "sync/atomic"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/agentcapabilities"+ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/internal/models"+ "github.com/rcourtman/pulse-go-rewrite/internal/monitoring"+ unified "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"+)++const metricAuthorityToken = "metric-authority-primary.12345678"+const metricAuthorityControl = "metric-authority-control.12345678"+const metricAuthorityID = "physical-disk:fixture-node:sda"++// Pause the real resource lookup after route admission. This is a normal+// provider boundary, not an auth stub, database lock or production test hook.+type metricAuthorityProvider struct {+ staticPhysicalDiskResourceStore+ entered chan struct{}+ release chan struct{}+ once sync.Once+ calls atomic.Int32+}++func (p *metricAuthorityProvider) GetAll() []unified.Resource {+ if p.calls.Add(1) <= 2 {+ p.entered <- struct{}{}+ <-p.release+ }+ return p.staticPhysicalDiskResourceStore.GetAll()+}+func (p *metricAuthorityProvider) ReadSnapshot() models.StateSnapshot { return models.StateSnapshot{} }+func (p *metricAuthorityProvider) UnifiedResourceSnapshot() ([]unified.Resource, time.Time) {+ return p.GetAll(), time.Now()+}+func (p *metricAuthorityProvider) unblock() { p.once.Do(func() { close(p.release) }) }++type metricAuthorityFixture struct {+ router *Router+ server *httptest.Server+ provider *metricAuthorityProvider+ path string+}++func newMetricAuthorityFixture(t *testing.T, entry string, scopes []string) *metricAuthorityFixture {+ t.Helper()+ setMockModeForTest(t, false)+ t.Setenv("PULSE_AUTH_BYPASS", "false")+ cfg := newTestConfigWithTokens(t,+ newTokenRecord(t, metricAuthorityToken, scopes, nil),+ newTokenRecord(t, metricAuthorityControl, []string{config.ScopeWildcard}, nil),+ )+ cfg.AuthUser, cfg.AuthPass = "operator", "fixture-nonempty-hash"+ cfg.ProxyAuthSecret, cfg.ProxyAuthUserHeader = "metric-authority-proxy-secret", "X-Remote-User"+ InitSessionStore(cfg.DataPath)+ p := &metricAuthorityProvider{+ staticPhysicalDiskResourceStore: staticPhysicalDiskResourceStore{resources: []unified.Resource{{+ ID: metricAuthorityID, Type: unified.ResourceTypePhysicalDisk, Name: "synthetic-disk",+ PhysicalDisk: &unified.PhysicalDiskMeta{Serial: "metric-authority-serial", Temperature: 37},+ MetricsTarget: &unified.MetricsTarget{ResourceType: "disk", ResourceID: metricAuthorityID},+ }}},+ entered: make(chan struct{}, 2), release: make(chan struct{}),+ }+ monitor := &monitoring.Monitor{}+ setUnexportedField(t, monitor, "state", models.NewState())+ setUnexportedField(t, monitor, "metricsHistory", monitoring.NewMetricsHistory(10, time.Hour))+ monitor.SetResourceStore(p)+ router := NewRouter(cfg, nil, nil, nil, nil, "test")+ router.monitor = monitor+ router.resourceHandlers.SetStateProvider(p)+ t.Cleanup(router.ShutdownResourceStores)+ path := "/api/metrics-store/history"+ q := url.Values{"resourceType": {"disk"}, "resourceId": {metricAuthorityID}, "metric": {"smart_temp"}, "range": {"1h"}}+ if entry == "agent" {+ path = agentcapabilities.MetricHistoryCapabilityPath+ q.Del("resourceType")+ }+ f := &metricAuthorityFixture{router: router, provider: p, path: path + "?" + q.Encode()}+ f.server = httptest.NewServer(router.Handler())+ t.Cleanup(f.server.Close)+ t.Cleanup(p.unblock)+ return f+}++func (f *metricAuthorityFixture) request(t *testing.T, raw, transport string) *http.Request {+ t.Helper()+ req, err := http.NewRequest(http.MethodGet, f.server.URL+f.path, nil)+ if err != nil {+ t.Fatal(err)+ }+ switch transport {+ case "header":+ req.Header.Set("X-API-Token", raw)+ case "bearer":+ req.Header.Set("Authorization", "Bearer "+raw)+ case "session":+ req.AddCookie(&http.Cookie{Name: cookieNameSession, Value: raw})+ case "proxy":+ req.Header.Set("X-Proxy-Secret", f.router.config.ProxyAuthSecret)+ req.Header.Set("X-Remote-User", "operator")+ default:+ t.Fatalf("unknown metric fixture transport %q", transport)+ }+ return req+}+func (f *metricAuthorityFixture) begin(t *testing.T, raw, transport string) <-chan historyAuthorityResult {+ t.Helper()+ req := f.request(t, raw, transport)+ result := make(chan historyAuthorityResult, 1)+ go func() { response, err := f.server.Client().Do(req); result <- historyAuthorityResult{response, err} }()+ select {+ case <-f.provider.entered:+ case got := <-result:+ if got.response != nil {+ got.response.Body.Close()+ }+ t.Fatalf("metric history returned before its inventory gate: %v", got.err)+ case <-time.After(5 * time.Second):+ t.Fatal("metric history did not reach its inventory gate")+ }+ return result+}++type metricAuthorityWire struct {+ Status int `json:"status"`+ Header http.Header `json:"header"`+ Body string `json:"body"`+}++func readMetricAuthorityResult(t *testing.T, result <-chan historyAuthorityResult, status int) metricAuthorityWire {+ t.Helper()+ select {+ case got := <-result:+ if got.err != nil {+ t.Fatal(got.err)+ }+ defer got.response.Body.Close()+ body, err := io.ReadAll(got.response.Body)+ if err != nil {+ t.Fatal(err)+ }+ if got.response.StatusCode != status {+ t.Errorf("metric history status = %d, want %d: %s", got.response.StatusCode, status, body)+ }+ if status != http.StatusOK {+ if string(body) != "Authentication required\n" || strings.Contains(got.response.Header.Get("Content-Type"), "json") || strings.Contains(string(body), metricAuthorityID) {+ t.Error("withdrawn history disclosed data or implied successful empty JSON")+ }+ return metricAuthorityWire{got.response.StatusCode, got.response.Header, string(body)}+ }+ var history struct {+ metricsHistoryResponse+ Metrics map[string][]struct {+ Timestamp int64 `json:"timestamp"`+ Value float64 `json:"value"`+ } `json:"metrics"`+ }+ if json.Unmarshal(body, &history) != nil {+ t.Fatal("invalid history JSON")+ }+ if history.Metric == "" {+ history.Points = history.Metrics["smart_temp"]+ }+ if history.ResourceId != metricAuthorityID || history.ResourceType != "disk" || history.Source != "live" || len(history.Points) != 1 || history.Points[0].Value != 37 {+ t.Errorf("authorised history lost its real temperature and existing envelope: %s", body)+ }+ if got.response.Header.Get("Content-Type") != "application/json" {+ t.Error("history media changed")+ }+ return metricAuthorityWire{got.response.StatusCode, got.response.Header, string(body)}+ case <-time.After(5 * time.Second):+ t.Fatal("metric history did not finish after inventory release")+ }+ return metricAuthorityWire{}+}++// Revocation uses the registered persisted DELETE, not only a fixture edit.+func (f *metricAuthorityFixture) revokeThroughHTTP(t *testing.T) {+ t.Helper()+ id := f.router.config.APITokens[0].ID+ req, err := http.NewRequest(http.MethodDelete, f.server.URL+"/api/security/tokens/"+id, nil)+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("X-API-Token", metricAuthorityControl)+ response, err := f.server.Client().Do(req)+ if err != nil {+ t.Fatal(err)+ }+ body, err := io.ReadAll(response.Body)+ response.Body.Close()+ if err != nil || response.StatusCode != http.StatusNoContent {+ t.Fatalf("token deletion = %d %s %v", response.StatusCode, body, err)+ }+ persisted, err := f.router.persistence.LoadAPITokens()+ if err != nil || len(persisted) != 1 || persisted[0].ID == id {+ t.Fatalf("metric token revocation not persisted: %v", err)+ }+}++func TestMetricHistoryWithdrawnSessionCannotDisclose(t *testing.T) {+ for _, entry := range []string{"chart", "agent"} {+ for _, change := range []string{"logout", "expired", "replaced", "principal", "store"} {+ t.Run(entry+"/"+change, func(t *testing.T) {+ f := newMetricAuthorityFixture(t, entry, []string{config.ScopeMonitoringRead})+ const raw = "metric-authority-session"+ GetSessionStore().CreateSession(raw, time.Hour, "fixture", "127.0.0.1", f.router.config.AuthUser)+ t.Cleanup(func() { GetSessionStore().DeleteSession(raw) })+ control := f.begin(t, metricAuthorityControl, "header")+ primary := f.begin(t, raw, "session")+ switch change {+ case "logout":+ logoutReadSession(t, f.server.URL, raw)+ case "store":+ InitSessionStore(t.TempDir())+ default:+ mutateReadSession(raw, change)+ }+ f.provider.unblock()+ readMetricAuthorityResult(t, primary, http.StatusUnauthorized)+ readMetricAuthorityResult(t, control, http.StatusOK)+ })+ }+ }+}++func TestMetricHistoryAllSeriesCannotDiscloseAfterLogout(t *testing.T) {+ for _, entry := range []string{"chart", "agent"} {+ t.Run(entry, func(t *testing.T) {+ f := newMetricAuthorityFixture(t, entry, []string{config.ScopeMonitoringRead})+ f.path = strings.ReplaceAll(f.path, "metric=smart_temp&", "")+ const raw = "metric-authority-all-series-session"+ GetSessionStore().CreateSession(raw, time.Hour, "fixture", "127.0.0.1", f.router.config.AuthUser)+ t.Cleanup(func() { GetSessionStore().DeleteSession(raw) })+ control := f.begin(t, metricAuthorityControl, "header")+ primary := f.begin(t, raw, "session")+ logoutReadSession(t, f.server.URL, raw)+ f.provider.unblock()+ readMetricAuthorityResult(t, primary, http.StatusUnauthorized)+ readMetricAuthorityResult(t, control, http.StatusOK)+ })+ }+}++func TestMetricHistoryKeepsInitialAdmissionGates(t *testing.T) {+ for _, entry := range []string{"chart", "agent"} {+ t.Run(entry, func(t *testing.T) {+ f := newMetricAuthorityFixture(t, entry, []string{config.ScopeSettingsRead})+ for _, token := range []string{"", metricAuthorityToken} {+ req := f.request(t, token, "header")+ response, err := f.server.Client().Do(req)+ if err != nil {+ t.Fatal(err)+ }+ response.Body.Close()+ want := http.StatusForbidden+ if token == "" {+ want = http.StatusUnauthorized+ }+ if response.StatusCode != want || f.provider.calls.Load() != 0 {+ t.Fatalf("initial history gate = %d, want %d; inventory calls = %d", response.StatusCode, want, f.provider.calls.Load())+ }+ }+ req := f.request(t, metricAuthorityControl, "header")+ req.Method = http.MethodPost+ response, err := f.server.Client().Do(req)+ if err != nil {+ t.Fatal(err)+ }+ response.Body.Close()+ if response.StatusCode != http.StatusMethodNotAllowed || f.provider.calls.Load() != 0 {+ t.Fatal("history method gate changed")+ }+ })+ }+}diff --git a/internal/api/route_inventory_test.go b/internal/api/route_inventory_test.goindex 01c8fbfdc6..9445efcd81 100644--- a/internal/api/route_inventory_test.go+++ b/internal/api/route_inventory_test.go@@ -1018,3 +1018,43 @@ func TestAgentHistoryPreservesAdmittedReaders(t *testing.T) {})}}++func TestMetricHistoryPreservesAdmittedReaders(t *testing.T) {+ for _, entry := range []string{"chart", "agent"} {+ for _, kind := range []string{"monitoring", "execution_removed", "scope_added", "wildcard", "session", "proxy"} {+ t.Run(entry+"/"+kind, func(t *testing.T) {+ scopes := []string{config.ScopeMonitoringRead, config.ScopeSettingsRead}+ if kind == "execution_removed" {+ scopes = append(scopes, config.ScopeAIExecute)+ }+ if kind == "wildcard" {+ scopes = []string{config.ScopeWildcard}+ }+ f := newMetricAuthorityFixture(t, entry, scopes)+ raw, transport := metricAuthorityToken, "bearer"+ if kind == "session" {+ raw, transport = "metric-authority-unchanged-session", "session"+ GetSessionStore().CreateSession(raw, time.Hour, "fixture", "127.0.0.1", f.router.config.AuthUser)+ t.Cleanup(func() { GetSessionStore().DeleteSession(raw) })+ } else if kind == "proxy" {+ transport = "proxy"+ }+ control := f.begin(t, metricAuthorityControl, "header")+ primary := f.begin(t, raw, transport)+ config.Mu.Lock()+ if kind == "session" || kind == "proxy" {+ f.router.config.RemoveAPIToken(f.router.config.APITokens[0].ID)+ } else {+ f.router.config.APITokens[0].Scopes = []string{config.ScopeMonitoringRead}+ if kind == "scope_added" {+ f.router.config.APITokens[0].Scopes = append(f.router.config.APITokens[0].Scopes, config.ScopeAIExecute)+ }+ }+ config.Mu.Unlock()+ f.provider.unblock()+ readMetricAuthorityResult(t, primary, http.StatusOK)+ readMetricAuthorityResult(t, control, http.StatusOK)+ })+ }+ }+}diff --git a/internal/api/router_routes_monitoring.go b/internal/api/router_routes_monitoring.goindex 33ac19ef32..cf704be924 100644--- a/internal/api/router_routes_monitoring.go+++ b/internal/api/router_routes_monitoring.go@@ -22,7 +22,11 @@ func (r *Router) registerMonitoringResourceRoutes(r.mux.HandleFunc("/api/charts/workloads", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleWorkloadCharts)))r.mux.HandleFunc("/api/charts/infrastructure", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleInfrastructureCharts)))r.mux.HandleFunc("/api/metrics-store/stats", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleMetricsStoreStats)))- r.mux.HandleFunc("/api/metrics-store/history", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleMetricsHistory)))+ // A history read can wait for inventory or its provider after admission.+ // Both entry points retain monitoring-read authority until disclosure;+ // the canonical agent lookup must be inside that lifetime too.+ r.mux.HandleFunc("/api/metrics-store/history", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead,+ retainTokenResponseAuthority(r.config, config.ScopeMonitoringRead, false, r.handleMetricsHistory))))r.mux.HandleFunc("/api/availability-history", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.handleAvailabilityHistory)))r.mux.HandleFunc("/api/recovery/points", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.recoveryHandlers.HandleListPoints)))r.mux.HandleFunc("/api/recovery/series", RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.recoveryHandlers.HandleListSeries)))@@ -132,7 +136,7 @@ func (r *Router) registerMonitoringResourceRoutes(// served by the same history read the charts use.r.mux.HandleFunc(agentcapabilities.MetricHistoryCapabilityPath, RequireAuth(r.config, RequireScope(config.ScopeMonitoringRead, r.withExternalAgentCapabilityActivity(agentcapabilities.GetMetricHistoryCapabilityName,- r.handleAgentMetricHistory,+ retainTokenResponseAuthority(r.config, config.ScopeMonitoringRead, false, r.handleAgentMetricHistory),))))// Content-safe Patrol control status: the shared Patrol, Assistant,// governed action, verification, Patrol control proof, and optional MCP