The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
feat(profiling): extract allowlisted pprof labels into samples (#237)
FrameAutomatadafbd95f9709
diff --git a/.gitignore b/.gitignoreindex 8af82201..e4b6896f 100644--- a/.gitignore+++ b/.gitignore@@ -18,6 +18,10 @@ coverage.**.coverprofileprofile.cov+# Generated pprof profiles (e.g. backend/tools/genpprof output)+*.pprof+*.pprof.gz+# Dependency directories (remove the comment below to include it)# vendor/diff --git a/backend/app/profiling/model.go b/backend/app/profiling/model.goindex 1d699a91..68934f15 100644--- a/backend/app/profiling/model.go+++ b/backend/app/profiling/model.go@@ -2,6 +2,7 @@ package profilingimport ("hash/fnv"+ "sort""time")@@ -17,6 +18,12 @@ var keptSampleTypes = map[string]string{"alloc_space": TypeHeapAllocSpace,}+const LabelEndpoint = "endpoint"++var allowedLabels = map[string]struct{}{+ LabelEndpoint: {},+}+func IsGauge(profileType string) bool {return profileType == TypeHeapInuseSpace}@@ -30,6 +37,7 @@ type Sample struct {Type stringStackHash uint64Value int64+ Labels map[string]string}type Meta struct {@@ -58,6 +66,42 @@ func HashFrames(frames []string) uint64 {return h.Sum64()}+func allowlistedLabels(raw map[string][]string) map[string]string {+ var out map[string]string+ for k, v := range raw {+ if _, ok := allowedLabels[k]; !ok {+ continue+ }+ if len(v) == 0 {+ continue+ }+ if out == nil {+ out = make(map[string]string)+ }+ out[k] = v[0]+ }+ return out+}++func labelFingerprint(labels map[string]string) uint64 {+ if len(labels) == 0 {+ return 0+ }+ keys := make([]string, 0, len(labels))+ for k := range labels {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ h := fnv.New64a()+ for _, k := range keys {+ _, _ = h.Write([]byte(k))+ _, _ = h.Write([]byte{0})+ _, _ = h.Write([]byte(labels[k]))+ _, _ = h.Write([]byte{0})+ }+ return h.Sum64()+}+func reverseInPlace(s []string) {for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {s[i], s[j] = s[j], s[i]diff --git a/backend/app/profiling/pprof_decoder.go b/backend/app/profiling/pprof_decoder.goindex 05d579a4..c60c235a 100644--- a/backend/app/profiling/pprof_decoder.go+++ b/backend/app/profiling/pprof_decoder.go@@ -15,6 +15,17 @@ type sampleKey struct {stackHash uint64}+type dedupKey struct {+ typ string+ stackHash uint64+ labelHash uint64+}++type dedupValue struct {+ value int64+ labels map[string]string+}+func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error) {p, err := profile.Parse(bytes.NewReader(payload))if err != nil {@@ -43,7 +54,7 @@ func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error)meta.End = meta.Start.Add(time.Duration(p.DurationNanos))}- values := make(map[sampleKey]int64)+ values := make(map[dedupKey]*dedupValue)stacks := make(map[uint64][]string)for _, s := range p.Sample {@@ -52,11 +63,19 @@ func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error)continue}hash := HashFrames(frames)+ sampleLabels := allowlistedLabels(s.Label)+ labelHash := labelFingerprint(sampleLabels)for _, k := range kept {if k.index >= len(s.Value) || s.Value[k.index] == 0 {continue}- values[sampleKey{typ: k.typ, stackHash: hash}] += s.Value[k.index]+ key := dedupKey{typ: k.typ, stackHash: hash, labelHash: labelHash}+ agg := values[key]+ if agg == nil {+ agg = &dedupValue{labels: sampleLabels}+ values[key] = agg+ }+ agg.value += s.Value[k.index]stacks[hash] = frames}}@@ -65,11 +84,12 @@ func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error)for hash, frames := range stacks {decoded.Stacks = append(decoded.Stacks, Stack{Hash: hash, Frames: frames})}- for key, v := range values {+ for key, agg := range values {decoded.Samples = append(decoded.Samples, Sample{Type: key.typ,StackHash: key.stackHash,- Value: v,+ Value: agg.value,+ Labels: agg.labels,})}diff --git a/backend/app/profiling/pprof_labels_test.go b/backend/app/profiling/pprof_labels_test.gonew file mode 100644index 00000000..967ba19f--- /dev/null+++ b/backend/app/profiling/pprof_labels_test.go@@ -0,0 +1,102 @@+package profiling++import (+ "testing"++ "github.com/google/pprof/profile"+ "github.com/google/uuid"+)++func labeledCPU(t *testing.T, samples []*profile.Sample, funcs []*profile.Function, locs []*profile.Location) []byte {+ return cpuProfile(t, 1_700_000_000_000_000_000, 1_000_000_000, samples, funcs, locs)+}++func TestPprofDecoder_KeepsOnlyAllowlistedLabels(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ work := fn(2, "main.work", "work.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ lWork := loc(2, profile.Line{Function: work, Line: 20})++ samples := []*profile.Sample{+ {+ Location: []*profile.Location{lWork, lMain},+ Value: []int64{1, 100},+ Label: map[string][]string{"endpoint": {"/checkout"}, "request_id": {"abc-123"}},+ },+ }+ d := decodeOne(t, labeledCPU(t, samples, []*profile.Function{main, work}, []*profile.Location{lMain, lWork}))++ cpu := sampleByType(d, TypeCPUNanos)+ if len(cpu) != 1 {+ t.Fatalf("expected 1 sample, got %d", len(cpu))+ }+ got := cpu[0].Labels+ if len(got) != 1 || got[LabelEndpoint] != "/checkout" {+ t.Errorf("labels = %v, want only {endpoint:/checkout} (high-cardinality request_id dropped)", got)+ }+}++func TestPprofDecoder_LabelsSeparateTheDedup(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ work := fn(2, "main.work", "work.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ lWork := loc(2, profile.Line{Function: work, Line: 20})++ mk := func(endpoint string, v int64) *profile.Sample {+ return &profile.Sample{+ Location: []*profile.Location{lWork, lMain},+ Value: []int64{1, v},+ Label: map[string][]string{"endpoint": {endpoint}},+ }+ }+ samples := []*profile.Sample{mk("/a", 100), mk("/b", 50), mk("/a", 25)}+ d := decodeOne(t, labeledCPU(t, samples, []*profile.Function{main, work}, []*profile.Location{lMain, lWork}))++ cpu := sampleByType(d, TypeCPUNanos)+ if len(cpu) != 2 {+ t.Fatalf("expected 2 samples (one per endpoint), got %d", len(cpu))+ }+ byEndpoint := map[string]int64{}+ for _, s := range cpu {+ byEndpoint[s.Labels[LabelEndpoint]] = s.Value+ }+ if byEndpoint["/a"] != 125 {+ t.Errorf("/a value = %d, want 125 (summed within the same endpoint)", byEndpoint["/a"])+ }+ if byEndpoint["/b"] != 50 {+ t.Errorf("/b value = %d, want 50", byEndpoint["/b"])+ }+ if len(d.Stacks) != 1 {+ t.Errorf("unique stacks = %d, want 1 (same stack, different labels still shares the stack row)", len(d.Stacks))+ }+}++func TestPprofDecoder_NoLabelsStaysEmpty(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ samples := []*profile.Sample{{Location: []*profile.Location{lMain}, Value: []int64{1, 100}}}+ d := decodeOne(t, labeledCPU(t, samples, []*profile.Function{main}, []*profile.Location{lMain}))++ cpu := sampleByType(d, TypeCPUNanos)+ if len(cpu) != 1 {+ t.Fatalf("expected 1 sample, got %d", len(cpu))+ }+ if len(cpu[0].Labels) != 0 {+ t.Errorf("labels = %v, want empty for a sample with no pprof labels", cpu[0].Labels)+ }+}++func TestBuildRows_CarriesSampleLabels(t *testing.T) {+ decoded := []Decoded{{+ Meta: Meta{ServiceName: "svc"},+ Stacks: []Stack{{Hash: 1, Frames: []string{"main.main"}}},+ Samples: []Sample{{Type: TypeCPUNanos, StackHash: 1, Value: 10, Labels: map[string]string{LabelEndpoint: "/x"}}},+ }}+ _, samples, _ := BuildRows(uuid.New(), decoded)+ if len(samples) != 1 {+ t.Fatalf("expected 1 ProfileSample, got %d", len(samples))+ }+ if samples[0].Labels[LabelEndpoint] != "/x" {+ t.Errorf("ProfileSample.Labels = %v, want {endpoint:/x}", samples[0].Labels)+ }+}diff --git a/backend/app/profiling/rows.go b/backend/app/profiling/rows.goindex f488ee7c..7b93fcf6 100644--- a/backend/app/profiling/rows.go+++ b/backend/app/profiling/rows.go@@ -66,6 +66,7 @@ func BuildRows(projectId uuid.UUID, decoded []Decoded) ([]models.ProfileStack, [End: d.Meta.End,StackHash: s.StackHash,Value: s.Value,+ Labels: s.Labels,ServerName: d.Meta.ServerName,AppVersion: d.Meta.AppVersion,TraceId: traceId,diff --git a/backend/tools/genpprof/README.md b/backend/tools/genpprof/README.mdnew file mode 100644index 00000000..a230c578--- /dev/null+++ b/backend/tools/genpprof/README.md@@ -0,0 +1,39 @@+# genpprof++Generates a synthetic CPU pprof for manually exercising the profiling label+pipeline (allowlist + label-separated dedup). The profile carries three CPU+samples on a single stack: two for `endpoint=/checkout` (100 + 25) and one for+`endpoint=/cart` (50), each also tagged with a high-cardinality `request_id`.++After ingest the decoder should keep only the allowlisted `endpoint` label,+drop `request_id`, and emit two samples (`/checkout`=125, `/cart`=50) sharing+one stack row.++## Generate++```bash+cd backend+go run ./tools/genpprof -out labeled.pprof # raw pprof+go run ./tools/genpprof -out labeled.pprof.gz -gzip # gzipped, ready to POST+```++## End-to-end against a running backend++```bash+# backend on :8082 (DB_TYPE=sqlite is the simplest local path)+go run ./tools/genpprof -out labeled.pprof.gz -gzip++curl -sS -X POST "http://localhost:8082/profiles/ingest?service=checkout-svc&serverName=pod-a&appVersion=9.9.9" \+ -H "Authorization: Bearer <PROJECT_TOKEN>" \+ -H "Content-Encoding: gzip" \+ --data-binary @labeled.pprof.gz++curl -sS -X POST "http://localhost:8082/profiles/flamegraph?projectId=<PROJECT_ID>" \+ -H "Authorization: Bearer <JWT>" \+ -H "Content-Type: application/json" \+ -d '{"serviceName":"checkout-svc","type":"go:profile_cpu:nanoseconds","labels":{"endpoint":"/checkout"}}'+```++The `/checkout`-filtered flame graph should weigh 125; dropping the `labels`+filter weighs 175 (both endpoints); a `request_id` filter matches nothing+because that key is never stored.diff --git a/backend/tools/genpprof/main.go b/backend/tools/genpprof/main.gonew file mode 100644index 00000000..b3929c1d--- /dev/null+++ b/backend/tools/genpprof/main.go@@ -0,0 +1,76 @@+package main++import (+ "compress/gzip"+ "flag"+ "fmt"+ "os"++ "github.com/google/pprof/profile"+)++func main() {+ out := flag.String("out", "labeled.pprof", "output file path")+ gz := flag.Bool("gzip", false, "gzip the output (sets the body for an /profiles/ingest POST with Content-Encoding: gzip)")+ flag.Parse()++ p := buildProfile()+ if err := p.CheckValid(); err != nil {+ fail(err)+ }++ f, err := os.Create(*out)+ if err != nil {+ fail(err)+ }+ defer f.Close()++ if *gz {+ w := gzip.NewWriter(f)+ if err := p.Write(w); err != nil {+ fail(err)+ }+ if err := w.Close(); err != nil {+ fail(err)+ }+ } else if err := p.Write(f); err != nil {+ fail(err)+ }++ fmt.Printf("wrote %s (gzip=%v)\n", *out, *gz)+ fmt.Println("expect after decode: 2 cpu samples on 1 stack -> endpoint /checkout=125, /cart=50; request_id dropped")+}++func fail(err error) {+ fmt.Fprintln(os.Stderr, "genpprof:", err)+ os.Exit(1)+}++func buildProfile() *profile.Profile {+ mainFn := &profile.Function{ID: 1, Name: "main.main", SystemName: "main.main", Filename: "main.go"}+ workFn := &profile.Function{ID: 2, Name: "main.work", SystemName: "main.work", Filename: "work.go"}+ lMain := &profile.Location{ID: 1, Line: []profile.Line{{Function: mainFn, Line: 10}}}+ lWork := &profile.Location{ID: 2, Line: []profile.Line{{Function: workFn, Line: 20}}}++ sample := func(endpoint string, v int64) *profile.Sample {+ return &profile.Sample{+ Location: []*profile.Location{lWork, lMain},+ Value: []int64{1, v},+ Label: map[string][]string{"endpoint": {endpoint}, "request_id": {"req-" + endpoint}},+ }+ }++ return &profile.Profile{+ SampleType: []*profile.ValueType{+ {Type: "samples", Unit: "count"},+ {Type: "cpu", Unit: "nanoseconds"},+ },+ PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},+ Period: 10_000_000,+ TimeNanos: 1_700_000_000_000_000_000,+ DurationNanos: 1_000_000_000,+ Sample: []*profile.Sample{sample("/checkout", 100), sample("/cart", 50), sample("/checkout", 25)},+ Function: []*profile.Function{mainFn, workFn},+ Location: []*profile.Location{lMain, lWork},+ }+}