Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reduce HTTP metric route-label intermediates
pulse-triage[bot]01788459f113
diff --git a/internal/api/http_metrics.go b/internal/api/http_metrics.goindex 03da7ec8ea..41d1a4bab7 100644--- a/internal/api/http_metrics.go+++ b/internal/api/http_metrics.go@@ -98,29 +98,43 @@ func normalizeRoute(path string) string {}// Strip query parameters.- if idx := strings.Index(path, "?"); idx >= 0 {+ if idx := strings.IndexByte(path, '?'); idx >= 0 {path = path[:idx]}- segments := strings.Split(path, "/")- normSegments := make([]string, 0, len(segments))- count := 0- for _, seg := range segments {+ // Only five non-empty segments enter the label. Keep those on the stack+ // instead of allocating slices for the entire (possibly much longer) path.+ var segments [5]string+ count, size := 0, 0+ for len(path) > 0 && count < len(segments) {+ seg := path+ if idx := strings.IndexByte(path, '/'); idx >= 0 {+ seg, path = path[:idx], path[idx+1:]+ } else {+ path = ""+ }if seg == "" {continue}+ seg = normalizeSegment(seg)+ segments[count] = seg+ size += 1 + len(seg)count++- if count > 5 {- break- }- normSegments = append(normSegments, normalizeSegment(seg))}- if len(normSegments) == 0 {+ if count == 0 {return "/"}- return "/" + strings.Join(normSegments, "/")+ // One owned string, including the leading slash. Returning a path prefix+ // would retain discarded query/token bytes in long-lived metric labels.+ var route strings.Builder+ route.Grow(size)+ for _, seg := range segments[:count] {+ route.WriteByte('/')+ route.WriteString(seg)+ }+ return route.String()}func normalizeSegment(seg string) string {diff --git a/internal/api/http_metrics_allocations_test.go b/internal/api/http_metrics_allocations_test.goindex ddc01b2ffe..18fda99393 100644--- a/internal/api/http_metrics_allocations_test.go+++ b/internal/api/http_metrics_allocations_test.go@@ -32,7 +32,7 @@ func isolatedRouteAllocationCheck(t *testing.T, measure func()) {func runRouteAllocationTest(t *testing.T, name string) {t.Helper()switch name {- case "TestNormalizeSegment_DoesNotAllocate", "TestNormalizeRoute_RootFastPathDoesNotAllocate":+ case "TestNormalizeSegment_DoesNotAllocate", "TestNormalizeRoute_RootFastPathDoesNotAllocate", "TestNormalizeRoute_BoundedAllocations":default:t.Fatalf("unknown route allocation measurement %q", name)}diff --git a/internal/api/http_metrics_route_test.go b/internal/api/http_metrics_route_test.gonew file mode 100644index 0000000000..41a100e2f3--- /dev/null+++ b/internal/api/http_metrics_route_test.go@@ -0,0 +1,241 @@+package api++import (+ "math/rand"+ "net/http"+ "net/http/httptest"+ "strconv"+ "strings"+ "sync"+ "testing"++ "github.com/prometheus/client_golang/prometheus"+)++// This deliberately keeps the original split/join algorithm and classifiers+// independent of the bounded scanner. In particular, UUID-like historically+// allows hex at the dash positions too; this optimization must not tighten it.+func referenceRouteLabel(path string) string {+ if idx := strings.IndexByte(path, '?'); idx >= 0 {+ path = path[:idx]+ }+ var labels []string+ for _, seg := range strings.Split(path, "/") {+ if seg == "" {+ continue+ }+ if len(labels) == 5 {+ break+ }+ numeric := true+ for _, r := range seg {+ if r < '0' || r > '9' {+ numeric = false+ }+ }+ uuid := len(seg) == 36+ for i, r := range seg {+ if r == '-' && (i == 8 || i == 13 || i == 18 || i == 23) {+ continue+ }+ if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {+ uuid = false+ }+ }+ switch {+ case numeric:+ seg = ":id"+ case uuid:+ seg = ":uuid"+ case len(seg) > 32:+ seg = ":token"+ }+ labels = append(labels, seg)+ }+ if len(labels) == 0 {+ return "/"+ }+ return "/" + strings.Join(labels, "/")+}++func routeLabelCorpus() []string {+ paths := []string{+ "", "/", "//", "?query/only", "/?query/only", "////?query/only",+ "api/resources", "/api///users/123//", "/a/b/c/d/e///ignored/99?query/only",+ "/a/b/c/d/e?query/only", "/a/b/c/d?e/f", "/café/123/١٢٣/node\xff",+ "/api/" + strings.Repeat("a", 32), "/api/" + strings.Repeat("é", 16),+ "/api/" + strings.Repeat("a", 33), "/api/" + strings.Repeat("é", 17),+ "/api/" + strings.Repeat("1", 64), "/api/" + strings.Repeat("a", 36),+ "/api/550E8400-E29B-41D4-A716-446655440000/status",+ "/a/b/c/d/e/" + strings.Repeat("discarded/", 2048),+ }+ segments := []string{"", "api", "v1", "123", "000", "-1", "12.3", "café", "node\xff", "123", "١٢٣",+ strings.Repeat("a", 32), strings.Repeat("a", 33), strings.Repeat("1", 36), strings.Repeat("a", 36),+ "550e8400-e29b-41d4-a716-446655440000", "550e8400-e29b-41d4-a716-44665544000!"}+ rng := rand.New(rand.NewSource(2576))+ for i := 0; i < 4096; i++ {+ parts := make([]string, rng.Intn(12))+ for j := range parts {+ parts[j] = segments[rng.Intn(len(segments))]+ }+ path := strings.Join(parts, "/")+ if i%2 == 0 {+ path = "/" + path + "/"+ }+ if i%3 == 0 {+ path += "?secret=never/a/label"+ }+ paths = append(paths, path)+ }+ return paths+}++func TestNormalizeRoute_Compatibility(t *testing.T) {+ for _, path := range routeLabelCorpus() {+ if got, want := normalizeRoute(path), referenceRouteLabel(path); got != want {+ t.Fatalf("normalizeRoute(%q) = %q, original = %q", path, got, want)+ }+ }+}++func FuzzNormalizeRouteCompatibility(f *testing.F) {+ for _, path := range routeLabelCorpus()[:20] {+ f.Add(path)+ }+ f.Fuzz(func(t *testing.T, path string) {+ if got, want := normalizeRoute(path), referenceRouteLabel(path); got != want {+ t.Fatalf("normalizeRoute(%q) = %q, original = %q", path, got, want)+ }+ })+}++func TestNormalizeRoute_BoundedAllocations(t *testing.T) {+ if !measureAllocsInIsolation(t) {+ return+ }+ for _, path := range []string{+ "/api/resources", "/api/users/12345", "/api/nodes/550e8400-e29b-41d4-a716-446655440000",+ "/api/v1/orgs/123/resources/456/metrics", "/api/metrics-store/history?secret=never/a/label",+ "/api/metrics-store/history", "/café/123/node\xff", "/auth/" + strings.Repeat("a", 64),+ "/a/b/c/d/e/" + strings.Repeat("discarded/", 2048),+ } {+ if allocs := testing.AllocsPerRun(100, func() { normalizeRouteSink = normalizeRoute(path) }); allocs > 1 {+ t.Errorf("normalizeRoute(%q) allocated %v times per call, want at most 1", path, allocs)+ }+ }+ for _, path := range []string{"", "/", "////", "?query/only", "/?query/only"} {+ if allocs := testing.AllocsPerRun(100, func() { normalizeRouteSink = normalizeRoute(path) }); allocs != 0 {+ t.Errorf("root label for %q allocated %v times per call, want 0", path, allocs)+ }+ }+}++func TestNormalizeRoute_ConcurrentAndRetainedLabels(t *testing.T) {+ paths := routeLabelCorpus()[:20]+ var workers sync.WaitGroup+ for range 16 {+ workers.Go(func() {+ labels := make([]string, 0, 64*len(paths))+ for range 64 {+ for _, path := range paths {+ labels = append(labels, normalizeRoute(path))+ }+ }+ for i, label := range labels {+ if want := referenceRouteLabel(paths[i%len(paths)]); label != want {+ t.Errorf("retained label %q changed; want %q", label, want)+ return+ }+ }+ })+ }+ workers.Wait()+}++func TestErrorHandler_NormalizedRouteMetrics(t *testing.T) {+ httpMetricsOnce.Do(initHTTPMetrics)+ cases := []struct {+ path, route, method string+ status int+ }{+ {"/metricroute-contract/users/", "/metricroute-contract/users/:id", "GET", 200},+ {"/metricroute-contract/auth/", "/metricroute-contract/auth/:token", "POST", 500},+ }+ registry := prometheus.NewRegistry()+ registry.MustRegister(apiRequestTotal, apiRequestDuration, apiRequestErrors)+ const requests = 64+ for _, c := range cases {+ status := strconv.Itoa(c.status)+ before := httpCounterValue(t, apiRequestTotal, c.method, c.route, status)+ observations := httpHistogramSampleCount(t, apiRequestDuration, c.method, c.route, status)+ errors := httpCounterValue(t, apiRequestErrors, c.method, c.route, classifyStatus(c.status))+ handler := ErrorHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(c.status)+ }))+ var workers sync.WaitGroup+ for i := range requests {+ workers.Go(func() {+ id := strconv.Itoa(i + 1)+ if c.status == 500 {+ id = strings.Repeat("token", 8) + id+ }+ req := httptest.NewRequest(c.method, c.path+id+"?secret="+id, nil)+ rec := httptest.NewRecorder()+ handler.ServeHTTP(rec, req)+ if rec.Code != c.status || rec.Header().Get("X-Request-ID") == "" {+ t.Errorf("response lost status or request ID: %d", rec.Code)+ }+ })+ }+ workers.Wait()+ if delta := httpCounterValue(t, apiRequestTotal, c.method, c.route, status) - before; delta != requests {+ t.Errorf("%s request count delta = %v, want %d", c.route, delta, requests)+ }+ if delta := httpHistogramSampleCount(t, apiRequestDuration, c.method, c.route, status) - observations; delta != requests {+ t.Errorf("%s histogram count delta = %v, want %d", c.route, delta, requests)+ }+ wantErrors := float64(0)+ if c.status >= 400 {+ wantErrors = requests+ }+ if delta := httpCounterValue(t, apiRequestErrors, c.method, c.route, classifyStatus(c.status)) - errors; delta != wantErrors {+ t.Errorf("%s error count delta = %v, want %v", c.route, delta, wantErrors)+ }+ }+ metrics, err := registry.Gather()+ if err != nil {+ t.Fatal(err)+ }+ series := 0+ for _, family := range metrics {+ if family.GetName() != "pulse_http_requests_total" {+ continue+ }+ for _, metric := range family.Metric {+ for _, label := range metric.Label {+ if label.GetName() == "route" && strings.HasPrefix(label.GetValue(), "/metricroute-contract/") {+ series+++ if label.GetValue() != cases[0].route && label.GetValue() != cases[1].route {+ t.Errorf("unexpected raw-ID/token/query route label %q", label.GetValue())+ }+ }+ }+ }+ }+ if series != len(cases) {+ t.Errorf("%d distinct requests created %d series, want %d", requests*len(cases), series, len(cases))+ }+}++func BenchmarkNormalizeRoute_DiscardedSuffix(b *testing.B) {+ for _, count := range []int{8, 2048} {+ b.Run(strconv.Itoa(count), func(b *testing.B) {+ path := "/a/b/c/d/e/" + strings.Repeat("discarded/", count)+ b.ReportAllocs()+ b.ResetTimer()+ for i := 0; i < b.N; i++ {+ _ = normalizeRoute(path)+ }+ })+ }+}