Skip to content

The only tool you need to know what is happening and how to fix it.

feat(profiling): extract allowlisted pprof labels into samples (#237)

FrameAutomatadafbd95f9709
diff --git a/.gitignore b/.gitignore
index 8af82201..e4b6896f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,10 @@ coverage.*
*.coverprofile
profile.cov
+# Generated pprof profiles (e.g. backend/tools/genpprof output)
+*.pprof
+*.pprof.gz
+
# Dependency directories (remove the comment below to include it)
# vendor/
diff --git a/backend/app/profiling/model.go b/backend/app/profiling/model.go
index 1d699a91..68934f15 100644
--- a/backend/app/profiling/model.go
+++ b/backend/app/profiling/model.go
@@ -2,6 +2,7 @@ package profiling
import (
"hash/fnv"
+ "sort"
"time"
)
@@ -17,6 +18,12 @@ var keptSampleTypes = map[string]string{
"alloc_space": TypeHeapAllocSpace,
}
+const LabelEndpoint = "endpoint"
+
+var allowedLabels = map[string]struct{}{
+ LabelEndpoint: {},
+}
+
func IsGauge(profileType string) bool {
return profileType == TypeHeapInuseSpace
}
@@ -30,6 +37,7 @@ type Sample struct {
Type string
StackHash uint64
Value int64
+ Labels map[string]string
}
type Meta struct {
@@ -58,6 +66,42 @@ func HashFrames(frames []string) uint64 {
return h.Sum64()
}
+func allowlistedLabels(raw map[string][]string) map[string]string {
+ var out map[string]string
+ for k, v := range raw {
+ if _, ok := allowedLabels[k]; !ok {
+ continue
+ }
+ if len(v) == 0 {
+ continue
+ }
+ if out == nil {
+ out = make(map[string]string)
+ }
+ out[k] = v[0]
+ }
+ return out
+}
+
+func labelFingerprint(labels map[string]string) uint64 {
+ if len(labels) == 0 {
+ return 0
+ }
+ keys := make([]string, 0, len(labels))
+ for k := range labels {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ h := fnv.New64a()
+ for _, k := range keys {
+ _, _ = h.Write([]byte(k))
+ _, _ = h.Write([]byte{0})
+ _, _ = h.Write([]byte(labels[k]))
+ _, _ = h.Write([]byte{0})
+ }
+ return h.Sum64()
+}
+
func reverseInPlace(s []string) {
for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
s[i], s[j] = s[j], s[i]
diff --git a/backend/app/profiling/pprof_decoder.go b/backend/app/profiling/pprof_decoder.go
index 05d579a4..c60c235a 100644
--- a/backend/app/profiling/pprof_decoder.go
+++ b/backend/app/profiling/pprof_decoder.go
@@ -15,6 +15,17 @@ type sampleKey struct {
stackHash uint64
}
+type dedupKey struct {
+ typ string
+ stackHash uint64
+ labelHash uint64
+}
+
+type dedupValue struct {
+ value int64
+ labels map[string]string
+}
+
func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error) {
p, err := profile.Parse(bytes.NewReader(payload))
if err != nil {
@@ -43,7 +54,7 @@ func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error)
meta.End = meta.Start.Add(time.Duration(p.DurationNanos))
}
- values := make(map[sampleKey]int64)
+ values := make(map[dedupKey]*dedupValue)
stacks := make(map[uint64][]string)
for _, s := range p.Sample {
@@ -52,11 +63,19 @@ func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error)
continue
}
hash := HashFrames(frames)
+ sampleLabels := allowlistedLabels(s.Label)
+ labelHash := labelFingerprint(sampleLabels)
for _, k := range kept {
if k.index >= len(s.Value) || s.Value[k.index] == 0 {
continue
}
- values[sampleKey{typ: k.typ, stackHash: hash}] += s.Value[k.index]
+ key := dedupKey{typ: k.typ, stackHash: hash, labelHash: labelHash}
+ agg := values[key]
+ if agg == nil {
+ agg = &dedupValue{labels: sampleLabels}
+ values[key] = agg
+ }
+ agg.value += s.Value[k.index]
stacks[hash] = frames
}
}
@@ -65,11 +84,12 @@ func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error)
for hash, frames := range stacks {
decoded.Stacks = append(decoded.Stacks, Stack{Hash: hash, Frames: frames})
}
- for key, v := range values {
+ for key, agg := range values {
decoded.Samples = append(decoded.Samples, Sample{
Type: key.typ,
StackHash: key.stackHash,
- Value: v,
+ Value: agg.value,
+ Labels: agg.labels,
})
}
diff --git a/backend/app/profiling/pprof_labels_test.go b/backend/app/profiling/pprof_labels_test.go
new file mode 100644
index 00000000..967ba19f
--- /dev/null
+++ b/backend/app/profiling/pprof_labels_test.go
@@ -0,0 +1,102 @@
+package profiling
+
+import (
+ "testing"
+
+ "github.com/google/pprof/profile"
+ "github.com/google/uuid"
+)
+
+func labeledCPU(t *testing.T, samples []*profile.Sample, funcs []*profile.Function, locs []*profile.Location) []byte {
+ return cpuProfile(t, 1_700_000_000_000_000_000, 1_000_000_000, samples, funcs, locs)
+}
+
+func TestPprofDecoder_KeepsOnlyAllowlistedLabels(t *testing.T) {
+ main := fn(1, "main.main", "main.go")
+ work := fn(2, "main.work", "work.go")
+ lMain := loc(1, profile.Line{Function: main, Line: 10})
+ lWork := loc(2, profile.Line{Function: work, Line: 20})
+
+ samples := []*profile.Sample{
+ {
+ Location: []*profile.Location{lWork, lMain},
+ Value: []int64{1, 100},
+ Label: map[string][]string{"endpoint": {"/checkout"}, "request_id": {"abc-123"}},
+ },
+ }
+ d := decodeOne(t, labeledCPU(t, samples, []*profile.Function{main, work}, []*profile.Location{lMain, lWork}))
+
+ cpu := sampleByType(d, TypeCPUNanos)
+ if len(cpu) != 1 {
+ t.Fatalf("expected 1 sample, got %d", len(cpu))
+ }
+ got := cpu[0].Labels
+ if len(got) != 1 || got[LabelEndpoint] != "/checkout" {
+ t.Errorf("labels = %v, want only {endpoint:/checkout} (high-cardinality request_id dropped)", got)
+ }
+}
+
+func TestPprofDecoder_LabelsSeparateTheDedup(t *testing.T) {
+ main := fn(1, "main.main", "main.go")
+ work := fn(2, "main.work", "work.go")
+ lMain := loc(1, profile.Line{Function: main, Line: 10})
+ lWork := loc(2, profile.Line{Function: work, Line: 20})
+
+ mk := func(endpoint string, v int64) *profile.Sample {
+ return &profile.Sample{
+ Location: []*profile.Location{lWork, lMain},
+ Value: []int64{1, v},
+ Label: map[string][]string{"endpoint": {endpoint}},
+ }
+ }
+ samples := []*profile.Sample{mk("/a", 100), mk("/b", 50), mk("/a", 25)}
+ d := decodeOne(t, labeledCPU(t, samples, []*profile.Function{main, work}, []*profile.Location{lMain, lWork}))
+
+ cpu := sampleByType(d, TypeCPUNanos)
+ if len(cpu) != 2 {
+ t.Fatalf("expected 2 samples (one per endpoint), got %d", len(cpu))
+ }
+ byEndpoint := map[string]int64{}
+ for _, s := range cpu {
+ byEndpoint[s.Labels[LabelEndpoint]] = s.Value
+ }
+ if byEndpoint["/a"] != 125 {
+ t.Errorf("/a value = %d, want 125 (summed within the same endpoint)", byEndpoint["/a"])
+ }
+ if byEndpoint["/b"] != 50 {
+ t.Errorf("/b value = %d, want 50", byEndpoint["/b"])
+ }
+ if len(d.Stacks) != 1 {
+ t.Errorf("unique stacks = %d, want 1 (same stack, different labels still shares the stack row)", len(d.Stacks))
+ }
+}
+
+func TestPprofDecoder_NoLabelsStaysEmpty(t *testing.T) {
+ main := fn(1, "main.main", "main.go")
+ lMain := loc(1, profile.Line{Function: main, Line: 10})
+ samples := []*profile.Sample{{Location: []*profile.Location{lMain}, Value: []int64{1, 100}}}
+ d := decodeOne(t, labeledCPU(t, samples, []*profile.Function{main}, []*profile.Location{lMain}))
+
+ cpu := sampleByType(d, TypeCPUNanos)
+ if len(cpu) != 1 {
+ t.Fatalf("expected 1 sample, got %d", len(cpu))
+ }
+ if len(cpu[0].Labels) != 0 {
+ t.Errorf("labels = %v, want empty for a sample with no pprof labels", cpu[0].Labels)
+ }
+}
+
+func TestBuildRows_CarriesSampleLabels(t *testing.T) {
+ decoded := []Decoded{{
+ Meta: Meta{ServiceName: "svc"},
+ Stacks: []Stack{{Hash: 1, Frames: []string{"main.main"}}},
+ Samples: []Sample{{Type: TypeCPUNanos, StackHash: 1, Value: 10, Labels: map[string]string{LabelEndpoint: "/x"}}},
+ }}
+ _, samples, _ := BuildRows(uuid.New(), decoded)
+ if len(samples) != 1 {
+ t.Fatalf("expected 1 ProfileSample, got %d", len(samples))
+ }
+ if samples[0].Labels[LabelEndpoint] != "/x" {
+ t.Errorf("ProfileSample.Labels = %v, want {endpoint:/x}", samples[0].Labels)
+ }
+}
diff --git a/backend/app/profiling/rows.go b/backend/app/profiling/rows.go
index f488ee7c..7b93fcf6 100644
--- a/backend/app/profiling/rows.go
+++ b/backend/app/profiling/rows.go
@@ -66,6 +66,7 @@ func BuildRows(projectId uuid.UUID, decoded []Decoded) ([]models.ProfileStack, [
End: d.Meta.End,
StackHash: s.StackHash,
Value: s.Value,
+ Labels: s.Labels,
ServerName: d.Meta.ServerName,
AppVersion: d.Meta.AppVersion,
TraceId: traceId,
diff --git a/backend/tools/genpprof/README.md b/backend/tools/genpprof/README.md
new file mode 100644
index 00000000..a230c578
--- /dev/null
+++ b/backend/tools/genpprof/README.md
@@ -0,0 +1,39 @@
+# genpprof
+
+Generates a synthetic CPU pprof for manually exercising the profiling label
+pipeline (allowlist + label-separated dedup). The profile carries three CPU
+samples on a single stack: two for `endpoint=/checkout` (100 + 25) and one for
+`endpoint=/cart` (50), each also tagged with a high-cardinality `request_id`.
+
+After ingest the decoder should keep only the allowlisted `endpoint` label,
+drop `request_id`, and emit two samples (`/checkout`=125, `/cart`=50) sharing
+one stack row.
+
+## Generate
+
+```bash
+cd backend
+go run ./tools/genpprof -out labeled.pprof # raw pprof
+go run ./tools/genpprof -out labeled.pprof.gz -gzip # gzipped, ready to POST
+```
+
+## End-to-end against a running backend
+
+```bash
+# backend on :8082 (DB_TYPE=sqlite is the simplest local path)
+go run ./tools/genpprof -out labeled.pprof.gz -gzip
+
+curl -sS -X POST "http://localhost:8082/profiles/ingest?service=checkout-svc&serverName=pod-a&appVersion=9.9.9" \
+ -H "Authorization: Bearer <PROJECT_TOKEN>" \
+ -H "Content-Encoding: gzip" \
+ --data-binary @labeled.pprof.gz
+
+curl -sS -X POST "http://localhost:8082/profiles/flamegraph?projectId=<PROJECT_ID>" \
+ -H "Authorization: Bearer <JWT>" \
+ -H "Content-Type: application/json" \
+ -d '{"serviceName":"checkout-svc","type":"go:profile_cpu:nanoseconds","labels":{"endpoint":"/checkout"}}'
+```
+
+The `/checkout`-filtered flame graph should weigh 125; dropping the `labels`
+filter weighs 175 (both endpoints); a `request_id` filter matches nothing
+because that key is never stored.
diff --git a/backend/tools/genpprof/main.go b/backend/tools/genpprof/main.go
new file mode 100644
index 00000000..b3929c1d
--- /dev/null
+++ b/backend/tools/genpprof/main.go
@@ -0,0 +1,76 @@
+package main
+
+import (
+ "compress/gzip"
+ "flag"
+ "fmt"
+ "os"
+
+ "github.com/google/pprof/profile"
+)
+
+func main() {
+ out := flag.String("out", "labeled.pprof", "output file path")
+ gz := flag.Bool("gzip", false, "gzip the output (sets the body for an /profiles/ingest POST with Content-Encoding: gzip)")
+ flag.Parse()
+
+ p := buildProfile()
+ if err := p.CheckValid(); err != nil {
+ fail(err)
+ }
+
+ f, err := os.Create(*out)
+ if err != nil {
+ fail(err)
+ }
+ defer f.Close()
+
+ if *gz {
+ w := gzip.NewWriter(f)
+ if err := p.Write(w); err != nil {
+ fail(err)
+ }
+ if err := w.Close(); err != nil {
+ fail(err)
+ }
+ } else if err := p.Write(f); err != nil {
+ fail(err)
+ }
+
+ fmt.Printf("wrote %s (gzip=%v)\n", *out, *gz)
+ fmt.Println("expect after decode: 2 cpu samples on 1 stack -> endpoint /checkout=125, /cart=50; request_id dropped")
+}
+
+func fail(err error) {
+ fmt.Fprintln(os.Stderr, "genpprof:", err)
+ os.Exit(1)
+}
+
+func buildProfile() *profile.Profile {
+ mainFn := &profile.Function{ID: 1, Name: "main.main", SystemName: "main.main", Filename: "main.go"}
+ workFn := &profile.Function{ID: 2, Name: "main.work", SystemName: "main.work", Filename: "work.go"}
+ lMain := &profile.Location{ID: 1, Line: []profile.Line{{Function: mainFn, Line: 10}}}
+ lWork := &profile.Location{ID: 2, Line: []profile.Line{{Function: workFn, Line: 20}}}
+
+ sample := func(endpoint string, v int64) *profile.Sample {
+ return &profile.Sample{
+ Location: []*profile.Location{lWork, lMain},
+ Value: []int64{1, v},
+ Label: map[string][]string{"endpoint": {endpoint}, "request_id": {"req-" + endpoint}},
+ }
+ }
+
+ return &profile.Profile{
+ SampleType: []*profile.ValueType{
+ {Type: "samples", Unit: "count"},
+ {Type: "cpu", Unit: "nanoseconds"},
+ },
+ PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},
+ Period: 10_000_000,
+ TimeNanos: 1_700_000_000_000_000_000,
+ DurationNanos: 1_000_000_000,
+ Sample: []*profile.Sample{sample("/checkout", 100), sample("/cart", 50), sample("/checkout", 25)},
+ Function: []*profile.Function{mainFn, workFn},
+ Location: []*profile.Location{lMain, lWork},
+ }
+}