Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Cover availability history authority through delayed body reads
pulse-triage[bot]79ea871b4f8a
diff --git a/internal/api/availability_history_authority_test.go b/internal/api/availability_history_authority_test.gonew file mode 100644index 0000000000..a0ed000b66--- /dev/null+++ b/internal/api/availability_history_authority_test.go@@ -0,0 +1,289 @@+package api++import (+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/rcourtman/pulse-go-rewrite/internal/config"+ "github.com/rcourtman/pulse-go-rewrite/pkg/metrics"+)++const availabilityAuthorityToken = "availability-reader-fixture.12345678"+const availabilityAuthorityControl = "availability-control-fixture.12345678"+const availabilityAuthorityTarget = "availability-history-private-fixture"+const availabilityAuthorityBody = `{"targetIds":["` + availabilityAuthorityTarget + `","missing"],"range":"24h"}`++// The request body is a natural wait after route authentication, before a+// history query. No auth stub, database lock or runtime test hook is needed.+type availabilityAuthorityBodyGate struct {+ io.Reader+ entered chan struct{}+ release chan struct{}+ once sync.Once+ open sync.Once+}++func (b *availabilityAuthorityBodyGate) Read(p []byte) (int, error) {+ b.once.Do(func() { close(b.entered); <-b.release })+ return b.Reader.Read(p)+}+func (b *availabilityAuthorityBodyGate) Close() error { return nil }+func (b *availabilityAuthorityBodyGate) unblock() { b.open.Do(func() { close(b.release) }) }++type availabilityAuthorityFixture struct {+ router *Router+ server *httptest.Server+}++func newAvailabilityAuthorityFixture(t *testing.T, scopes []string) *availabilityAuthorityFixture {+ t.Helper()+ setMockModeForTest(t, false)+ t.Setenv("PULSE_AUTH_BYPASS", "false")+ cfg := newTestConfigWithTokens(t,+ newTokenRecord(t, availabilityAuthorityToken, scopes, nil),+ newTokenRecord(t, availabilityAuthorityControl, []string{config.ScopeWildcard}, nil),+ )+ cfg.AuthUser, cfg.AuthPass = "operator", "fixture-nonempty-hash"+ InitSessionStore(cfg.DataPath)+ router := NewRouter(cfg, nil, nil, nil, nil, "test")+ t.Cleanup(router.ShutdownResourceStores)+ monitor, _, _ := newTestMonitor(t)+ store, err := metrics.NewStore(metrics.DefaultConfig(t.TempDir()))+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { store.Close() })+ setUnexportedField(t, monitor, "metricsStore", store)+ router.monitor = monitor+ target := config.NormalizeAvailabilityTarget(config.AvailabilityTarget{+ ID: availabilityAuthorityTarget, Name: "Synthetic gateway", Address: "gateway.invalid",+ TargetKind: config.AvailabilityTargetService, Protocol: config.AvailabilityProbeHTTPS, Enabled: true,+ })+ if err := router.persistence.SaveAvailabilityTargets([]config.AvailabilityTarget{target}); err != nil {+ t.Fatal(err)+ }+ at := time.Now().UTC().Truncate(time.Minute).Add(-10 * time.Minute)+ latency := int64(24)+ if err := store.WriteAvailabilityObservationSync(metrics.AvailabilityObservation{+ ObservationID: "availability-authority-fixture", TargetID: target.ID, ConfigRevision: target.ConfigRevision,+ Outcome: metrics.AvailabilityReachable, ObservedAt: at, TimelineAt: at, IngestedAt: at,+ ValidFor: 5 * time.Minute, ExecutionSource: metrics.AvailabilitySourceLocal, LatencyMillis: &latency,+ }); err != nil {+ t.Fatal(err)+ }+ f := &availabilityAuthorityFixture{router: router, server: httptest.NewServer(router.Handler())}+ t.Cleanup(f.server.Close)+ return f+}++func (f *availabilityAuthorityFixture) request(t *testing.T, raw, transport, body string) *http.Request {+ t.Helper()+ req := httptest.NewRequest(http.MethodPost, "/api/availability-history", strings.NewReader(body))+ req.Header.Set("Content-Type", "application/json")+ switch transport {+ case "header":+ req.Header.Set("X-API-Token", raw)+ case "bearer":+ req.Header.Set("Authorization", "Bearer "+raw)+ case "session":+ req.AddCookie(&http.Cookie{Name: cookieNameSession, Value: raw})+ req.Header.Set("X-CSRF-Token", generateCSRFToken(raw))+ }+ return req+}++func (f *availabilityAuthorityFixture) begin(t *testing.T, raw, transport, body string) (<-chan *httptest.ResponseRecorder, func()) {+ t.Helper()+ req := f.request(t, raw, transport, body)+ gate := &availabilityAuthorityBodyGate{Reader: req.Body, entered: make(chan struct{}), release: make(chan struct{})}+ req.Body = gate+ result := make(chan *httptest.ResponseRecorder, 1)+ go func() {+ w := httptest.NewRecorder()+ f.router.Handler().ServeHTTP(w, req)+ result <- w+ }()+ t.Cleanup(gate.unblock)+ select {+ case <-gate.entered:+ case w := <-result:+ t.Fatalf("history returned before body admission: %d %s", w.Code, w.Body.String())+ case <-time.After(5 * time.Second):+ t.Fatal("history did not reach admitted body read")+ }+ return result, gate.unblock+}++func readAvailabilityAuthorityResult(t *testing.T, result <-chan *httptest.ResponseRecorder, status int) *httptest.ResponseRecorder {+ t.Helper()+ select {+ case w := <-result:+ if w.Code != status {+ t.Errorf("history status = %d, want %d: %s", w.Code, status, w.Body.String())+ }+ if status == http.StatusUnauthorized {+ if w.Body.String() != "Authentication required\n" || strings.Contains(w.Header().Get("Content-Type"), "json") || strings.Contains(w.Body.String(), availabilityAuthorityTarget) {+ t.Error("withdrawn history disclosed data or implied successful empty JSON")+ }+ }+ if status == http.StatusOK {+ var response availabilityHistoryResponse+ if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {+ t.Fatal(err)+ }+ if len(response.Targets) != 2 {+ t.Fatalf("lost batch members: %s", w.Body.String())+ }+ history := response.Targets[0]+ if history.TargetID != availabilityAuthorityTarget || history.Summary == nil || history.Summary.ReachableSeconds != 300 || len(history.Buckets) == 0 || response.Targets[1].Error == nil || response.Targets[1].Error.Code != "not_found" {+ t.Errorf("history or per-target absence changed: %s", w.Body.String())+ }+ if w.Header().Get("Content-Type") != "application/json" {+ t.Error("history media changed")+ }+ }+ return w+ case <-time.After(5 * time.Second):+ t.Fatal("history did not finish after body release")+ }+ return nil+}++func (f *availabilityAuthorityFixture) revoke(t *testing.T) {+ t.Helper()+ id := f.router.config.APITokens[0].ID+ req, err := http.NewRequest(http.MethodDelete, f.server.URL+"/api/security/tokens/"+id, nil)+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("X-API-Token", availabilityAuthorityControl)+ response, err := f.server.Client().Do(req)+ if err != nil {+ t.Fatal(err)+ }+ defer response.Body.Close()+ if response.StatusCode != http.StatusNoContent {+ t.Fatalf("token DELETE = %d", response.StatusCode)+ }+ tokens, err := f.router.persistence.LoadAPITokens()+ if err != nil || len(tokens) != 1 || tokens[0].ID == id {+ t.Fatalf("token withdrawal not persisted: %v", err)+ }+}++func TestAvailabilityHistoryWithdrawnTokenCannotDisclose(t *testing.T) {+ for _, transport := range []string{"header", "bearer"} {+ for _, change := range []string{"revoked", "expired", "scope", "organization", "credential", "principal"} {+ t.Run(transport+"/"+change, func(t *testing.T) {+ f := newAvailabilityAuthorityFixture(t, []string{config.ScopeMonitoringRead})+ control, releaseControl := f.begin(t, availabilityAuthorityControl, "header", availabilityAuthorityBody)+ primary, release := f.begin(t, availabilityAuthorityToken, transport, availabilityAuthorityBody)+ if change == "revoked" {+ f.revoke(t)+ } else {+ config.Mu.Lock()+ record := &f.router.config.APITokens[0]+ switch change {+ case "expired":+ past := time.Now().Add(-time.Second)+ record.ExpiresAt = &past+ case "scope":+ record.Scopes = []string{config.ScopeSettingsRead}+ case "organization":+ record.OrgIDs = []string{"other-org"}+ case "credential":+ record.ID = "replacement-availability-credential"+ case "principal":+ record.Metadata = map[string]string{apiTokenMetadataOwnerUserID: "replacement-availability-principal"}+ }+ config.Mu.Unlock()+ }+ release()+ releaseControl()+ withdrawn := readAvailabilityAuthorityResult(t, primary, http.StatusUnauthorized)+ allowed := readAvailabilityAuthorityResult(t, control, http.StatusOK)+ if transport == "header" && change == "revoked" {+ for name, response := range map[string]*httptest.ResponseRecorder{"withdrawn": withdrawn, "control": allowed} {+ wire, _ := json.Marshal(map[string]any{"status": response.Code, "header": response.Header(), "body": response.Body.String()})+ t.Logf("AVAILABILITY_WIRE_%s %s", name, wire)+ }+ }+ fresh := httptest.NewRecorder()+ f.router.Handler().ServeHTTP(fresh, f.request(t, availabilityAuthorityToken, transport, availabilityAuthorityBody))+ if fresh.Code == http.StatusOK {+ t.Error("fresh read reused withdrawn authority")+ }+ })+ }+ }+}++func TestAvailabilityHistoryWithdrawnSessionCannotDisclose(t *testing.T) {+ for _, change := range []string{"logout", "expired", "replaced", "principal", "store"} {+ t.Run(change, func(t *testing.T) {+ f := newAvailabilityAuthorityFixture(t, []string{config.ScopeMonitoringRead})+ const raw = "synthetic-availability-session"+ GetSessionStore().CreateSession(raw, time.Hour, "fixture", "127.0.0.1", f.router.config.AuthUser)+ control, releaseControl := f.begin(t, availabilityAuthorityControl, "header", availabilityAuthorityBody)+ primary, release := f.begin(t, raw, "session", availabilityAuthorityBody)+ if change == "logout" {+ logoutReadSession(t, f.server.URL, raw)+ } else if change == "store" {+ InitSessionStore(t.TempDir())+ } else {+ mutateReadSession(raw, change)+ }+ release()+ releaseControl()+ readAvailabilityAuthorityResult(t, primary, http.StatusUnauthorized)+ readAvailabilityAuthorityResult(t, control, http.StatusOK)+ })+ }+}++func TestAvailabilityHistoryWithdrawalWithholdsErrors(t *testing.T) {+ for _, body := range []string{`{`, `{"targetIds":["one"],"range":"14d"}`, availabilityAuthorityBody} {+ t.Run(body, func(t *testing.T) {+ f := newAvailabilityAuthorityFixture(t, []string{config.ScopeMonitoringRead})+ // The last case exercises an admitted provider-unavailable result.+ if body == availabilityAuthorityBody {+ f.router.monitor = nil+ }+ primary, release := f.begin(t, availabilityAuthorityToken, "header", body)+ f.revoke(t)+ release()+ readAvailabilityAuthorityResult(t, primary, http.StatusUnauthorized)+ })+ }+}++func TestAvailabilityHistoryKeepsAdmissionAndRecovery(t *testing.T) {+ f := newAvailabilityAuthorityFixture(t, []string{config.ScopeSettingsRead})+ for _, tc := range []struct {+ raw, transport string+ status int+ }{+ {"", "header", http.StatusUnauthorized},+ {availabilityAuthorityToken, "header", http.StatusForbidden},+ {availabilityAuthorityControl, "header", http.StatusOK},+ } {+ w := httptest.NewRecorder()+ f.router.Handler().ServeHTTP(w, f.request(t, tc.raw, tc.transport, availabilityAuthorityBody))+ if w.Code != tc.status {+ t.Fatalf("admission = %d, want %d: %s", w.Code, tc.status, w.Body.String())+ }+ }+ // Restoring monitoring-read authorises a new request, not the old one.+ config.Mu.Lock()+ f.router.config.APITokens[0].Scopes = []string{config.ScopeMonitoringRead}+ config.Mu.Unlock()+ result, release := f.begin(t, availabilityAuthorityToken, "header", availabilityAuthorityBody)+ release()+ readAvailabilityAuthorityResult(t, result, http.StatusOK)+}