The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
feat(profiling): pprof decoder + ClickHouse storage foundation (PR 1) (#209)
FrameAutomata53cf2a9c788f
diff --git a/backend/app/migrations/ch/0065_create_profiling_stacks.up.sql b/backend/app/migrations/ch/0065_create_profiling_stacks.up.sqlnew file mode 100644index 00000000..d553d5bb--- /dev/null+++ b/backend/app/migrations/ch/0065_create_profiling_stacks.up.sql@@ -0,0 +1,11 @@+CREATE TABLE IF NOT EXISTS profiling_stacks+(+ `project_id` UUID,+ `service_name` LowCardinality(String) DEFAULT '',+ `stack_hash` UInt64,+ `stack` Array(LowCardinality(String)),+ `last_seen` DateTime64(3)+)+ENGINE = ReplacingMergeTree(last_seen)+ORDER BY (project_id, service_name, stack_hash)+SETTINGS index_granularity = 8192diff --git a/backend/app/migrations/ch/0066_create_profiling_samples.up.sql b/backend/app/migrations/ch/0066_create_profiling_samples.up.sqlnew file mode 100644index 00000000..27b9c811--- /dev/null+++ b/backend/app/migrations/ch/0066_create_profiling_samples.up.sql@@ -0,0 +1,21 @@+CREATE TABLE IF NOT EXISTS profiling_samples+(+ `project_id` UUID,+ `profile_id` UUID,+ `service_name` LowCardinality(String) DEFAULT '',+ `type` LowCardinality(String) DEFAULT '',+ `start_time` DateTime64(3),+ `end_time` DateTime64(3),+ `stack_hash` UInt64,+ `value` Int64 CODEC(ZSTD(1)),+ `labels` Map(LowCardinality(String), String),+ `server_name` LowCardinality(String) DEFAULT '',+ `app_version` LowCardinality(String) DEFAULT '',+ `trace_id` String DEFAULT '',+ `span_id` String DEFAULT '',+ INDEX idx_stack_hash stack_hash TYPE bloom_filter(0.001) GRANULARITY 1+)+ENGINE = MergeTree+PARTITION BY toYYYYMMDD(start_time)+ORDER BY (project_id, type, service_name, start_time)+SETTINGS index_granularity = 8192diff --git a/backend/app/migrations/ch/0067_create_profiles.up.sql b/backend/app/migrations/ch/0067_create_profiles.up.sqlnew file mode 100644index 00000000..bd7244ff--- /dev/null+++ b/backend/app/migrations/ch/0067_create_profiles.up.sql@@ -0,0 +1,24 @@+CREATE TABLE IF NOT EXISTS profiles+(+ `id` UUID,+ `project_id` UUID,+ `recorded_at` DateTime64(3),+ `duration` Int64 DEFAULT 0,+ `service_name` LowCardinality(String) DEFAULT '',+ `profile_type` LowCardinality(String) DEFAULT '',+ `sample_count` UInt64 DEFAULT 0,+ `total_value` Int64 DEFAULT 0,+ `server_name` LowCardinality(String) DEFAULT '',+ `app_version` LowCardinality(String) DEFAULT '',+ `attributes` String DEFAULT '{}',+ `storage_key` String DEFAULT '',+ `trace_id` String DEFAULT '',+ `span_id` String DEFAULT '',+ `distributed_trace_id` Nullable(UUID) DEFAULT NULL,+ INDEX idx_id id TYPE bloom_filter(0.001) GRANULARITY 1,+ INDEX idx_distributed_trace_id distributed_trace_id TYPE bloom_filter(0.001) GRANULARITY 1+)+ENGINE = MergeTree+PARTITION BY toYYYYMMDD(recorded_at)+ORDER BY (project_id, recorded_at, service_name, profile_type)+SETTINGS index_granularity = 8192diff --git a/backend/app/migrations/sqlite_telemetry/0012_create_profiling.up.sql b/backend/app/migrations/sqlite_telemetry/0012_create_profiling.up.sqlnew file mode 100644index 00000000..bd27c1f8--- /dev/null+++ b/backend/app/migrations/sqlite_telemetry/0012_create_profiling.up.sql@@ -0,0 +1,42 @@+CREATE TABLE IF NOT EXISTS profiling_stacks (+ project_id TEXT NOT NULL,+ service_name TEXT NOT NULL DEFAULT '',+ stack_hash INTEGER NOT NULL,+ stack TEXT NOT NULL DEFAULT '[]',+ last_seen DATETIME NOT NULL,+ UNIQUE(project_id, service_name, stack_hash)+);+CREATE TABLE IF NOT EXISTS profiling_samples (+ project_id TEXT NOT NULL,+ profile_id TEXT NOT NULL,+ service_name TEXT NOT NULL DEFAULT '',+ type TEXT NOT NULL DEFAULT '',+ start_time DATETIME NOT NULL,+ end_time DATETIME NOT NULL,+ stack_hash INTEGER NOT NULL,+ value INTEGER NOT NULL DEFAULT 0,+ labels TEXT NOT NULL DEFAULT '{}',+ server_name TEXT NOT NULL DEFAULT '',+ app_version TEXT NOT NULL DEFAULT '',+ trace_id TEXT NOT NULL DEFAULT '',+ span_id TEXT NOT NULL DEFAULT ''+);+CREATE INDEX IF NOT EXISTS idx_profiling_samples_query ON profiling_samples(project_id, type, service_name, start_time);+CREATE TABLE IF NOT EXISTS profiles (+ id TEXT NOT NULL,+ project_id TEXT NOT NULL,+ recorded_at DATETIME NOT NULL,+ duration INTEGER NOT NULL DEFAULT 0,+ service_name TEXT NOT NULL DEFAULT '',+ profile_type TEXT NOT NULL DEFAULT '',+ sample_count INTEGER NOT NULL DEFAULT 0,+ total_value INTEGER NOT NULL DEFAULT 0,+ server_name TEXT NOT NULL DEFAULT '',+ app_version TEXT NOT NULL DEFAULT '',+ attributes TEXT NOT NULL DEFAULT '{}',+ storage_key TEXT NOT NULL DEFAULT '',+ trace_id TEXT NOT NULL DEFAULT '',+ span_id TEXT NOT NULL DEFAULT '',+ distributed_trace_id TEXT DEFAULT NULL+);+CREATE INDEX IF NOT EXISTS idx_profiles_project_recorded ON profiles(project_id, recorded_at);diff --git a/backend/app/models/profile.model.go b/backend/app/models/profile.model.gonew file mode 100644index 00000000..58e7a8df--- /dev/null+++ b/backend/app/models/profile.model.go@@ -0,0 +1,49 @@+package models++import (+ "time"++ "github.com/google/uuid"+)++type ProfileStack struct {+ ProjectId uuid.UUID `json:"projectId" ch:"project_id"`+ ServiceName string `json:"serviceName" ch:"service_name"`+ StackHash uint64 `json:"stackHash" ch:"stack_hash"`+ Stack []string `json:"stack" ch:"stack"`+ LastSeen time.Time `json:"lastSeen" ch:"last_seen"`+}++type ProfileSample struct {+ ProjectId uuid.UUID `json:"projectId" ch:"project_id"`+ ProfileId uuid.UUID `json:"profileId" ch:"profile_id"`+ ServiceName string `json:"serviceName" ch:"service_name"`+ Type string `json:"type" ch:"type"`+ Start time.Time `json:"start" ch:"start_time"`+ End time.Time `json:"end" ch:"end_time"`+ StackHash uint64 `json:"stackHash" ch:"stack_hash"`+ Value int64 `json:"value" ch:"value"`+ Labels map[string]string `json:"labels" ch:"labels"`+ ServerName string `json:"serverName" ch:"server_name"`+ AppVersion string `json:"appVersion" ch:"app_version"`+ TraceId string `json:"traceId" ch:"trace_id"`+ SpanId string `json:"spanId" ch:"span_id"`+}++type Profile struct {+ Id uuid.UUID `json:"id" ch:"id"`+ ProjectId uuid.UUID `json:"projectId" ch:"project_id"`+ RecordedAt time.Time `json:"recordedAt" ch:"recorded_at"`+ Duration time.Duration `json:"duration" ch:"duration"`+ ServiceName string `json:"serviceName" ch:"service_name"`+ ProfileType string `json:"profileType" ch:"profile_type"`+ SampleCount uint64 `json:"sampleCount" ch:"sample_count"`+ TotalValue int64 `json:"totalValue" ch:"total_value"`+ ServerName string `json:"serverName" ch:"server_name"`+ AppVersion string `json:"appVersion" ch:"app_version"`+ Attributes map[string]string `json:"attributes" ch:"attributes"`+ StorageKey string `json:"storageKey" ch:"storage_key"`+ TraceId string `json:"traceId" ch:"trace_id"`+ SpanId string `json:"spanId" ch:"span_id"`+ DistributedTraceId *uuid.UUID `json:"distributedTraceId,omitempty" ch:"distributed_trace_id"`+}diff --git a/backend/app/profiling/decoder.go b/backend/app/profiling/decoder.gonew file mode 100644index 00000000..44c075fa--- /dev/null+++ b/backend/app/profiling/decoder.go@@ -0,0 +1,19 @@+package profiling++import (+ "time"++ "github.com/google/uuid"+)++type IngestContext struct {+ ProjectId uuid.UUID+ DefaultServiceName string+ ServerName string+ AppVersion string+ ReceivedAt time.Time+}++type Decoder interface {+ Decode(ctx IngestContext, payload []byte) ([]Decoded, error)+}diff --git a/backend/app/profiling/model.go b/backend/app/profiling/model.gonew file mode 100644index 00000000..61e1e950--- /dev/null+++ b/backend/app/profiling/model.go@@ -0,0 +1,58 @@+package profiling++import (+ "hash/fnv"+ "time"++ "github.com/google/uuid"+)++const (+ TypeCPUNanos = "go:profile_cpu:nanoseconds"+ TypeHeapInuseSpace = "go:profile_heap:inuse_space"+ TypeHeapAllocSpace = "go:profile_heap:alloc_space"+)++var keptSampleTypes = map[string]string{+ "cpu": TypeCPUNanos,+ "inuse_space": TypeHeapInuseSpace,+ "alloc_space": TypeHeapAllocSpace,+}++type Stack struct {+ Hash uint64+ Frames []string+}++type Sample struct {+ Type string+ StackHash uint64+ Value int64+}++type Meta struct {+ ProfileId uuid.UUID+ ServiceName string+ Start time.Time+ End time.Time+ ServerName string+ AppVersion string+ Attributes map[string]string+ TraceId *string+ SpanId *string+}++type Decoded struct {+ Meta Meta+ Stacks []Stack+ Samples []Sample+}++func HashFrames(frames []string) uint64 {+ h := fnv.New64a()+ for _, f := range frames {+ _, _ = h.Write([]byte(f))+ _, _ = h.Write([]byte{0})+ }+ return h.Sum64()+}diff --git a/backend/app/profiling/pprof_decoder.go b/backend/app/profiling/pprof_decoder.gonew file mode 100644index 00000000..f7228b03--- /dev/null+++ b/backend/app/profiling/pprof_decoder.go@@ -0,0 +1,102 @@+package profiling++import (+ "bytes"+ "fmt"+ "time"++ "github.com/google/pprof/profile"+ "github.com/google/uuid"+)++type PprofDecoder struct{}++type sampleKey struct {+ typ string+ stackHash uint64+}++func (PprofDecoder) Decode(ctx IngestContext, payload []byte) ([]Decoded, error) {+ p, err := profile.Parse(bytes.NewReader(payload))+ if err != nil {+ return nil, fmt.Errorf("profiling: parse pprof: %w", err)+ }++ type keptType struct {+ index int+ typ string+ }+ var kept []keptType+ for i, st := range p.SampleType {+ if internal, ok := keptSampleTypes[st.Type]; ok {+ kept = append(kept, keptType{index: i, typ: internal})+ }+ }++ meta := Meta{+ ProfileId: uuid.New(),+ ServiceName: ctx.DefaultServiceName,+ ServerName: ctx.ServerName,+ AppVersion: ctx.AppVersion,+ Start: profileStart(p, ctx.ReceivedAt),+ }+ meta.End = meta.Start+ if p.DurationNanos > 0 {+ meta.End = meta.Start.Add(time.Duration(p.DurationNanos))+ }++ values := make(map[sampleKey]int64)+ stacks := make(map[uint64][]string)++ for _, s := range p.Sample {+ frames := rootFirstFrames(s)+ if len(frames) == 0 {+ continue+ }+ hash := HashFrames(frames)+ for _, k := range kept {+ if k.index >= len(s.Value) || s.Value[k.index] == 0 {+ continue+ }+ values[sampleKey{typ: k.typ, stackHash: hash}] += s.Value[k.index]+ stacks[hash] = frames+ }+ }++ decoded := Decoded{Meta: meta}+ for hash, frames := range stacks {+ decoded.Stacks = append(decoded.Stacks, Stack{Hash: hash, Frames: frames})+ }+ for key, v := range values {+ decoded.Samples = append(decoded.Samples, Sample{+ Type: key.typ,+ StackHash: key.stackHash,+ Value: v,+ })+ }++ return []Decoded{decoded}, nil+}++func profileStart(p *profile.Profile, fallback time.Time) time.Time {+ if p.TimeNanos > 0 {+ return time.Unix(0, p.TimeNanos).UTC()+ }+ return fallback+}++func rootFirstFrames(s *profile.Sample) []string {+ var leafToRoot []string+ for _, loc := range s.Location {+ for _, line := range loc.Line {+ if line.Function == nil {+ continue+ }+ leafToRoot = append(leafToRoot, line.Function.Name)+ }+ }+ for i, j := 0, len(leafToRoot)-1; i < j; i, j = i+1, j-1 {+ leafToRoot[i], leafToRoot[j] = leafToRoot[j], leafToRoot[i]+ }+ return leafToRoot+}diff --git a/backend/app/profiling/pprof_decoder_test.go b/backend/app/profiling/pprof_decoder_test.gonew file mode 100644index 00000000..940826c3--- /dev/null+++ b/backend/app/profiling/pprof_decoder_test.go@@ -0,0 +1,372 @@+package profiling++import (+ "bytes"+ "testing"+ "time"++ "github.com/google/pprof/profile"+ "github.com/google/uuid"+)++var testIngest = IngestContext{+ ProjectId: uuid.MustParse("00000000-0000-0000-0000-000000000001"),+ DefaultServiceName: "checkout",+ ServerName: "pod-abc",+ AppVersion: "1.2.3",+ ReceivedAt: time.Unix(1_700_000_000, 0).UTC(),+}++func fn(id uint64, name, file string) *profile.Function {+ return &profile.Function{ID: id, Name: name, SystemName: name, Filename: file}+}++func loc(id uint64, lines ...profile.Line) *profile.Location {+ return &profile.Location{ID: id, Line: lines}+}++func writeProfile(t *testing.T, p *profile.Profile) []byte {+ t.Helper()+ if err := p.CheckValid(); err != nil {+ t.Fatalf("invalid synthetic profile: %v", err)+ }+ var buf bytes.Buffer+ if err := p.Write(&buf); err != nil {+ t.Fatalf("write profile: %v", err)+ }+ return buf.Bytes()+}++func cpuProfile(t *testing.T, timeNanos, durationNanos int64, samples []*profile.Sample, funcs []*profile.Function, locs []*profile.Location) []byte {+ p := &profile.Profile{+ SampleType: []*profile.ValueType{+ {Type: "samples", Unit: "count"},+ {Type: "cpu", Unit: "nanoseconds"},+ },+ PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"},+ Period: 10_000_000,+ TimeNanos: timeNanos,+ DurationNanos: durationNanos,+ Sample: samples,+ Function: funcs,+ Location: locs,+ }+ return writeProfile(t, p)+}++func decodeOne(t *testing.T, payload []byte) Decoded {+ t.Helper()+ out, err := PprofDecoder{}.Decode(testIngest, payload)+ if err != nil {+ t.Fatalf("Decode error: %v", err)+ }+ if len(out) != 1 {+ t.Fatalf("expected 1 decoded profile, got %d", len(out))+ }+ return out[0]+}++func sampleByType(d Decoded, typ string) []Sample {+ var out []Sample+ for _, s := range d.Samples {+ if s.Type == typ {+ out = append(out, s)+ }+ }+ return out+}++func stackByHash(d Decoded, h uint64) (Stack, bool) {+ for _, s := range d.Stacks {+ if s.Hash == h {+ return s, true+ }+ }+ return Stack{}, false+}++func TestPprofDecoder_CPU_ExplodesDistinctStacks(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ work := fn(2, "main.work", "work.go")+ idle := fn(3, "main.idle", "idle.go")++ lMain := loc(1, profile.Line{Function: main, Line: 10})+ lWork := loc(2, profile.Line{Function: work, Line: 20})+ lIdle := loc(3, profile.Line{Function: idle, Line: 30})++ samples := []*profile.Sample{+ {Location: []*profile.Location{lWork, lMain}, Value: []int64{1, 300}},+ {Location: []*profile.Location{lIdle, lMain}, Value: []int64{1, 100}},+ }+ payload := cpuProfile(t, 1_700_000_000_000_000_000, 5_000_000_000, samples,+ []*profile.Function{main, work, idle}, []*profile.Location{lMain, lWork, lIdle})++ d := decodeOne(t, payload)++ cpu := sampleByType(d, TypeCPUNanos)+ if len(cpu) != 2 {+ t.Fatalf("expected 2 cpu samples, got %d", len(cpu))+ }+ if len(d.Stacks) != 2 {+ t.Fatalf("expected 2 unique stacks, got %d", len(d.Stacks))+ }++ wantWork := HashFrames([]string{"main.main", "main.work"})+ wantIdle := HashFrames([]string{"main.main", "main.idle"})+ gotValues := map[uint64]int64{}+ for _, s := range cpu {+ gotValues[s.StackHash] = s.Value+ }+ if gotValues[wantWork] != 300 {+ t.Errorf("main->work cpu value = %d, want 300", gotValues[wantWork])+ }+ if gotValues[wantIdle] != 100 {+ t.Errorf("main->idle cpu value = %d, want 100", gotValues[wantIdle])+ }++ st, ok := stackByHash(d, wantWork)+ if !ok {+ t.Fatalf("stack for main->work not present")+ }+ if got := st.Frames; len(got) != 2 || got[0] != "main.main" || got[1] != "main.work" {+ t.Errorf("frames = %v, want [main.main main.work] (root-first)", got)+ }+}++func TestPprofDecoder_DedupesIdenticalStacks(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ work := fn(2, "main.work", "work.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ lWork := loc(2, profile.Line{Function: work, Line: 20})++ samples := []*profile.Sample{+ {Location: []*profile.Location{lWork, lMain}, Value: []int64{1, 200}},+ {Location: []*profile.Location{lWork, lMain}, Value: []int64{1, 50}},+ }+ payload := cpuProfile(t, 1_700_000_000_000_000_000, 1_000_000_000, samples,+ []*profile.Function{main, work}, []*profile.Location{lMain, lWork})++ d := decodeOne(t, payload)++ cpu := sampleByType(d, TypeCPUNanos)+ if len(cpu) != 1 {+ t.Fatalf("expected 1 deduped cpu sample, got %d", len(cpu))+ }+ if cpu[0].Value != 250 {+ t.Errorf("summed value = %d, want 250", cpu[0].Value)+ }+ if len(d.Stacks) != 1 {+ t.Errorf("expected 1 unique stack, got %d", len(d.Stacks))+ }+}++func TestPprofDecoder_Heap_KeepsSpaceTypesOnly(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ alloc := fn(2, "main.allocate", "alloc.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ lAlloc := loc(2, profile.Line{Function: alloc, Line: 20})++ p := &profile.Profile{+ SampleType: []*profile.ValueType{+ {Type: "alloc_objects", Unit: "count"},+ {Type: "alloc_space", Unit: "bytes"},+ {Type: "inuse_objects", Unit: "count"},+ {Type: "inuse_space", Unit: "bytes"},+ },+ PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"},+ Period: 524288,+ TimeNanos: 1_700_000_000_000_000_000,+ Sample: []*profile.Sample{+ {Location: []*profile.Location{lAlloc, lMain}, Value: []int64{5, 4096, 2, 2048}},+ },+ Function: []*profile.Function{main, alloc},+ Location: []*profile.Location{lMain, lAlloc},+ }+ d := decodeOne(t, writeProfile(t, p))++ inuse := sampleByType(d, TypeHeapInuseSpace)+ allocS := sampleByType(d, TypeHeapAllocSpace)+ if len(inuse) != 1 || inuse[0].Value != 2048 {+ t.Errorf("inuse_space = %+v, want one sample value 2048", inuse)+ }+ if len(allocS) != 1 || allocS[0].Value != 4096 {+ t.Errorf("alloc_space = %+v, want one sample value 4096", allocS)+ }+ for _, s := range d.Samples {+ if s.Type != TypeHeapInuseSpace && s.Type != TypeHeapAllocSpace {+ t.Errorf("unexpected sample type emitted: %q", s.Type)+ }+ }+}++func TestPprofDecoder_InlinedFramesExpand(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ printf := fn(2, "fmt.Printf", "print.go")+ memcpy := fn(3, "runtime.memmove", "memmove.go")++ lMain := loc(1, profile.Line{Function: main, Line: 10})+ lInlined := loc(2,+ profile.Line{Function: memcpy, Line: 5},+ profile.Line{Function: printf, Line: 99},+ )+ samples := []*profile.Sample{+ {Location: []*profile.Location{lInlined, lMain}, Value: []int64{1, 100}},+ }+ payload := cpuProfile(t, 1_700_000_000_000_000_000, 1_000_000_000, samples,+ []*profile.Function{main, printf, memcpy}, []*profile.Location{lMain, lInlined})++ d := decodeOne(t, payload)+ if len(d.Stacks) != 1 {+ t.Fatalf("expected 1 stack, got %d", len(d.Stacks))+ }+ want := []string{"main.main", "fmt.Printf", "runtime.memmove"}+ got := d.Stacks[0].Frames+ if len(got) != len(want) {+ t.Fatalf("frames = %v, want %v", got, want)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("frames = %v, want %v", got, want)+ }+ }+}++func TestPprofDecoder_Metadata(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ start := int64(1_700_000_000_000_000_000)+ dur := int64(30_000_000_000)+ samples := []*profile.Sample{{Location: []*profile.Location{lMain}, Value: []int64{1, 100}}}+ payload := cpuProfile(t, start, dur, samples, []*profile.Function{main}, []*profile.Location{lMain})++ d := decodeOne(t, payload)+ if d.Meta.ServiceName != "checkout" {+ t.Errorf("service name = %q, want checkout", d.Meta.ServiceName)+ }+ if d.Meta.ServerName != "pod-abc" || d.Meta.AppVersion != "1.2.3" {+ t.Errorf("server/version not carried from ingest context: %+v", d.Meta)+ }+ wantStart := time.Unix(0, start).UTC()+ if !d.Meta.Start.Equal(wantStart) {+ t.Errorf("start = %v, want %v", d.Meta.Start.UTC(), wantStart)+ }+ if !d.Meta.End.Equal(wantStart.Add(30 * time.Second)) {+ t.Errorf("end = %v, want start+30s", d.Meta.End.UTC())+ }+ if d.Meta.ProfileId == uuid.Nil {+ t.Errorf("expected a non-nil ProfileId")+ }+}++func TestPprofDecoder_DropsZeroValueSamples(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ live := fn(2, "main.live", "live.go")+ freed := fn(3, "main.freed", "freed.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ lLive := loc(2, profile.Line{Function: live, Line: 20})+ lFreed := loc(3, profile.Line{Function: freed, Line: 30})++ p := &profile.Profile{+ SampleType: []*profile.ValueType{+ {Type: "alloc_objects", Unit: "count"},+ {Type: "alloc_space", Unit: "bytes"},+ {Type: "inuse_objects", Unit: "count"},+ {Type: "inuse_space", Unit: "bytes"},+ },+ PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"},+ Period: 524288,+ TimeNanos: 1_700_000_000_000_000_000,+ Sample: []*profile.Sample{+ {Location: []*profile.Location{lLive, lMain}, Value: []int64{10, 8192, 5, 4096}},+ {Location: []*profile.Location{lFreed, lMain}, Value: []int64{10, 8192, 0, 0}},+ },+ Function: []*profile.Function{main, live, freed},+ Location: []*profile.Location{lMain, lLive, lFreed},+ }+ d := decodeOne(t, writeProfile(t, p))++ inuse := sampleByType(d, TypeHeapInuseSpace)+ if len(inuse) != 1 {+ t.Fatalf("expected 1 inuse_space sample (zero dropped), got %d", len(inuse))+ }+ wantLive := HashFrames([]string{"main.main", "main.live"})+ if inuse[0].StackHash != wantLive || inuse[0].Value != 4096 {+ t.Errorf("inuse sample = %+v, want live stack value 4096", inuse[0])+ }+ if got := len(sampleByType(d, TypeHeapAllocSpace)); got != 2 {+ t.Errorf("alloc_space samples = %d, want 2", got)+ }+}++func TestPprofDecoder_UnsupportedTypesYieldNoSamples(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ p := &profile.Profile{+ SampleType: []*profile.ValueType{{Type: "goroutine", Unit: "count"}},+ PeriodType: &profile.ValueType{Type: "goroutine", Unit: "count"},+ Period: 1,+ TimeNanos: 1_700_000_000_000_000_000,+ Sample: []*profile.Sample{{Location: []*profile.Location{lMain}, Value: []int64{7}}},+ Function: []*profile.Function{main},+ Location: []*profile.Location{lMain},+ }+ out, err := PprofDecoder{}.Decode(testIngest, writeProfile(t, p))+ if err != nil {+ t.Fatalf("unexpected error on unsupported profile: %v", err)+ }+ if len(out) != 1 {+ t.Fatalf("expected 1 decoded profile, got %d", len(out))+ }+ if len(out[0].Samples) != 0 {+ t.Errorf("expected 0 samples for unsupported types, got %d", len(out[0].Samples))+ }+}++func TestPprofDecoder_ZeroDurationEndEqualsStart(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ start := int64(1_700_000_000_000_000_000)+ p := &profile.Profile{+ SampleType: []*profile.ValueType{{Type: "inuse_space", Unit: "bytes"}},+ PeriodType: &profile.ValueType{Type: "space", Unit: "bytes"},+ Period: 524288,+ TimeNanos: start,+ Sample: []*profile.Sample{{Location: []*profile.Location{lMain}, Value: []int64{4096}}},+ Function: []*profile.Function{main},+ Location: []*profile.Location{lMain},+ }+ d := decodeOne(t, writeProfile(t, p))+ if !d.Meta.End.Equal(d.Meta.Start) {+ t.Errorf("End=%v Start=%v; want equal for zero-duration profile", d.Meta.End, d.Meta.Start)+ }+}++func TestPprofDecoder_NoTimestampFallsBackToReceivedAt(t *testing.T) {+ main := fn(1, "main.main", "main.go")+ lMain := loc(1, profile.Line{Function: main, Line: 10})+ samples := []*profile.Sample{{Location: []*profile.Location{lMain}, Value: []int64{1, 100}}}+ payload := cpuProfile(t, 0, 0, samples, []*profile.Function{main}, []*profile.Location{lMain})++ d := decodeOne(t, payload)+ if !d.Meta.Start.Equal(testIngest.ReceivedAt) {+ t.Errorf("Start=%v, want fallback to ReceivedAt=%v", d.Meta.Start, testIngest.ReceivedAt)+ }+}++func TestPprofDecoder_RejectsGarbage(t *testing.T) {+ if _, err := (PprofDecoder{}).Decode(testIngest, []byte("not a pprof profile")); err == nil {+ t.Fatalf("expected error decoding garbage, got nil")+ }+}++func TestHashFrames(t *testing.T) {+ a := HashFrames([]string{"main.main", "main.work"})+ b := HashFrames([]string{"main.main", "main.work"})+ c := HashFrames([]string{"main.work", "main.main"})+ if a != b {+ t.Errorf("identical frame sequences hashed differently")+ }+ if a == c {+ t.Errorf("frame order must affect the hash")+ }+}diff --git a/backend/app/repositories/profile.repository.go b/backend/app/repositories/profile.repository.gonew file mode 100644index 00000000..0658e07d--- /dev/null+++ b/backend/app/repositories/profile.repository.go@@ -0,0 +1,83 @@+//go:build pgch++package repositories++import (+ "context"+ "encoding/json"++ "github.com/tracewayapp/traceway/backend/app/chdb"+ "github.com/tracewayapp/traceway/backend/app/models"+)++type profileRepository struct{}++func (r *profileRepository) InsertStacksAsync(ctx context.Context, stacks []models.ProfileStack) error {+ if len(stacks) == 0 {+ return nil+ }+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),+ "INSERT INTO profiling_stacks (project_id, service_name, stack_hash, stack, last_seen)")+ if err != nil {+ return err+ }+ for _, s := range stacks {+ if err := batch.Append(s.ProjectId, s.ServiceName, s.StackHash, s.Stack, s.LastSeen); err != nil {+ return err+ }+ }+ return batch.Send()+}++func (r *profileRepository) InsertSamplesAsync(ctx context.Context, samples []models.ProfileSample) error {+ if len(samples) == 0 {+ return nil+ }+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),+ "INSERT INTO profiling_samples (project_id, profile_id, service_name, type, start_time, end_time, stack_hash, value, labels, server_name, app_version, trace_id, span_id)")+ if err != nil {+ return err+ }+ for _, s := range samples {+ labels := s.Labels+ if labels == nil {+ labels = map[string]string{}+ }+ if err := batch.Append(+ s.ProjectId, s.ProfileId, s.ServiceName, s.Type, s.Start, s.End,+ s.StackHash, s.Value, labels, s.ServerName, s.AppVersion, s.TraceId, s.SpanId,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+}++func (r *profileRepository) InsertProfilesAsync(ctx context.Context, profiles []models.Profile) error {+ if len(profiles) == 0 {+ return nil+ }+ batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),+ "INSERT INTO profiles (id, project_id, recorded_at, duration, service_name, profile_type, sample_count, total_value, server_name, app_version, attributes, storage_key, trace_id, span_id, distributed_trace_id)")+ if err != nil {+ return err+ }+ for _, p := range profiles {+ attributesJSON := "{}"+ if len(p.Attributes) != 0 {+ if b, err := json.Marshal(p.Attributes); err == nil {+ attributesJSON = string(b)+ }+ }+ if err := batch.Append(+ p.Id, p.ProjectId, p.RecordedAt, int64(p.Duration), p.ServiceName, p.ProfileType,+ p.SampleCount, p.TotalValue, p.ServerName, p.AppVersion, attributesJSON, p.StorageKey,+ p.TraceId, p.SpanId, p.DistributedTraceId,+ ); err != nil {+ return err+ }+ }+ return batch.Send()+}++var ProfileRepository = profileRepository{}diff --git a/backend/app/repositories/profile.repository_sqlite.go b/backend/app/repositories/profile.repository_sqlite.gonew file mode 100644index 00000000..948c607c--- /dev/null+++ b/backend/app/repositories/profile.repository_sqlite.go@@ -0,0 +1,130 @@+//go:build !pgch++package repositories++import (+ "context"+ "encoding/json"++ "github.com/tracewayapp/lit/v2"+ "github.com/tracewayapp/traceway/backend/app/db"+ "github.com/tracewayapp/traceway/backend/app/models"+)++type profileRepository struct{}++func (r *profileRepository) InsertStacksAsync(ctx context.Context, stacks []models.ProfileStack) error {+ if len(stacks) == 0 {+ return nil+ }+ tx, err := db.TelemetryDB.BeginTx(ctx, nil)+ if err != nil {+ return err+ }+ defer tx.Rollback()++ for _, s := range stacks {+ stackJSON, err := json.Marshal(s.Stack)+ if err != nil {+ return err+ }+ query, args, err := lit.ParseNamedQuery(db.Driver,+ "INSERT OR REPLACE INTO profiling_stacks (project_id, service_name, stack_hash, stack, last_seen) VALUES (:project_id, :service_name, :stack_hash, :stack, :last_seen)",+ lit.P{+ "project_id": s.ProjectId,+ "service_name": s.ServiceName,+ "stack_hash": int64(s.StackHash),+ "stack": string(stackJSON),+ "last_seen": NewSQLiteTime(s.LastSeen),+ })+ if err != nil {+ return err+ }+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {+ return err+ }+ }+ return tx.Commit()+}++func (r *profileRepository) InsertSamplesAsync(ctx context.Context, samples []models.ProfileSample) error {+ if len(samples) == 0 {+ return nil+ }+ tx, err := db.TelemetryDB.BeginTx(ctx, nil)+ if err != nil {+ return err+ }+ defer tx.Rollback()++ for _, s := range samples {+ labelsVal, _ := NewSQLiteJSONMap(s.Labels).Value()+ query, args, err := lit.ParseNamedQuery(db.Driver,+ "INSERT INTO profiling_samples (project_id, profile_id, service_name, type, start_time, end_time, stack_hash, value, labels, server_name, app_version, trace_id, span_id) VALUES (:project_id, :profile_id, :service_name, :type, :start_time, :end_time, :stack_hash, :value, :labels, :server_name, :app_version, :trace_id, :span_id)",+ lit.P{+ "project_id": s.ProjectId,+ "profile_id": s.ProfileId,+ "service_name": s.ServiceName,+ "type": s.Type,+ "start_time": NewSQLiteTime(s.Start),+ "end_time": NewSQLiteTime(s.End),+ "stack_hash": int64(s.StackHash),+ "value": s.Value,+ "labels": labelsVal,+ "server_name": s.ServerName,+ "app_version": s.AppVersion,+ "trace_id": s.TraceId,+ "span_id": s.SpanId,+ })+ if err != nil {+ return err+ }+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {+ return err+ }+ }+ return tx.Commit()+}++func (r *profileRepository) InsertProfilesAsync(ctx context.Context, profiles []models.Profile) error {+ if len(profiles) == 0 {+ return nil+ }+ tx, err := db.TelemetryDB.BeginTx(ctx, nil)+ if err != nil {+ return err+ }+ defer tx.Rollback()++ for _, p := range profiles {+ attributesVal, _ := NewSQLiteJSONMap(p.Attributes).Value()+ query, args, err := lit.ParseNamedQuery(db.Driver,+ "INSERT INTO profiles (id, project_id, recorded_at, duration, service_name, profile_type, sample_count, total_value, server_name, app_version, attributes, storage_key, trace_id, span_id, distributed_trace_id) VALUES (:id, :project_id, :recorded_at, :duration, :service_name, :profile_type, :sample_count, :total_value, :server_name, :app_version, :attributes, :storage_key, :trace_id, :span_id, :distributed_trace_id)",+ lit.P{+ "id": p.Id,+ "project_id": p.ProjectId,+ "recorded_at": NewSQLiteTime(p.RecordedAt),+ "duration": int64(p.Duration),+ "service_name": p.ServiceName,+ "profile_type": p.ProfileType,+ "sample_count": int64(p.SampleCount),+ "total_value": p.TotalValue,+ "server_name": p.ServerName,+ "app_version": p.AppVersion,+ "attributes": attributesVal,+ "storage_key": p.StorageKey,+ "trace_id": p.TraceId,+ "span_id": p.SpanId,+ "distributed_trace_id": p.DistributedTraceId,+ })+ if err != nil {+ return err+ }+ if _, err := tx.ExecContext(ctx, query, args...); err != nil {+ return err+ }+ }+ return tx.Commit()+}++var ProfileRepository = profileRepository{}diff --git a/backend/app/repositories/profile_repository_test.go b/backend/app/repositories/profile_repository_test.gonew file mode 100644index 00000000..4850a8eb--- /dev/null+++ b/backend/app/repositories/profile_repository_test.go@@ -0,0 +1,101 @@+//go:build !pgch++package repositories++import (+ "context"+ "encoding/json"+ "testing"+ "time"++ "github.com/google/uuid"+ "github.com/tracewayapp/traceway/backend/app/db"+ "github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/profiling"+)++func TestProfileRepository_InsertAndReadBack(t *testing.T) {+ setupTestDB(t)+ ctx := context.Background()+ projectId := uuid.New()+ profileId := uuid.New()+ now := time.Now().UTC().Truncate(time.Millisecond)++ hash := uint64(0xFFFFFFFFFFFFFFF1)+ frames := []string{"main.main", "main.work"}++ stack := models.ProfileStack{+ ProjectId: projectId, ServiceName: "checkout",+ StackHash: hash, Stack: frames, LastSeen: now,+ }+ if err := ProfileRepository.InsertStacksAsync(ctx, []models.ProfileStack{stack}); err != nil {+ t.Fatalf("InsertStacksAsync: %v", err)+ }+ stack.LastSeen = now.Add(time.Minute)+ if err := ProfileRepository.InsertStacksAsync(ctx, []models.ProfileStack{stack}); err != nil {+ t.Fatalf("InsertStacksAsync (dedup): %v", err)+ }++ sample := models.ProfileSample{+ ProjectId: projectId, ProfileId: profileId, ServiceName: "checkout",+ Type: profiling.TypeCPUNanos, Start: now, End: now.Add(30 * time.Second),+ StackHash: hash, Value: 300, ServerName: "pod-a", AppVersion: "1.2.3",+ }+ if err := ProfileRepository.InsertSamplesAsync(ctx, []models.ProfileSample{sample}); err != nil {+ t.Fatalf("InsertSamplesAsync: %v", err)+ }++ prof := models.Profile{+ Id: profileId, ProjectId: projectId, RecordedAt: now, Duration: 30 * time.Second,+ ServiceName: "checkout", ProfileType: profiling.TypeCPUNanos,+ SampleCount: 1, TotalValue: 300, ServerName: "pod-a", AppVersion: "1.2.3",+ }+ if err := ProfileRepository.InsertProfilesAsync(ctx, []models.Profile{prof}); err != nil {+ t.Fatalf("InsertProfilesAsync: %v", err)+ }++ var stackCount int+ if err := db.TelemetryDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM profiling_stacks").Scan(&stackCount); err != nil {+ t.Fatalf("count stacks: %v", err)+ }+ if stackCount != 1 {+ t.Errorf("expected 1 deduped stack, got %d", stackCount)+ }++ var gotHash int64+ var gotStackJSON string+ if err := db.TelemetryDB.QueryRowContext(ctx,+ "SELECT stack_hash, stack FROM profiling_stacks LIMIT 1").Scan(&gotHash, &gotStackJSON); err != nil {+ t.Fatalf("read stack: %v", err)+ }+ if uint64(gotHash) != hash {+ t.Errorf("stack_hash round-trip = %d, want %d", uint64(gotHash), hash)+ }+ var gotFrames []string+ if err := json.Unmarshal([]byte(gotStackJSON), &gotFrames); err != nil {+ t.Fatalf("decode stack json: %v", err)+ }+ if len(gotFrames) != 2 || gotFrames[0] != "main.main" || gotFrames[1] != "main.work" {+ t.Errorf("frames = %v, want [main.main main.work]", gotFrames)+ }++ var gotType string+ var gotValue, gotSampleHash int64+ if err := db.TelemetryDB.QueryRowContext(ctx,+ "SELECT type, value, stack_hash FROM profiling_samples LIMIT 1").Scan(&gotType, &gotValue, &gotSampleHash); err != nil {+ t.Fatalf("read sample: %v", err)+ }+ if gotType != profiling.TypeCPUNanos || gotValue != 300 || uint64(gotSampleHash) != hash {+ t.Errorf("sample = (%s,%d,%d), want (%s,300,%d)", gotType, gotValue, uint64(gotSampleHash), profiling.TypeCPUNanos, hash)+ }++ var gotTotal int64+ var gotProfileType string+ if err := db.TelemetryDB.QueryRowContext(ctx,+ "SELECT profile_type, total_value FROM profiles WHERE id = ?", profileId.String()).Scan(&gotProfileType, &gotTotal); err != nil {+ t.Fatalf("read profile: %v", err)+ }+ if gotProfileType != profiling.TypeCPUNanos || gotTotal != 300 {+ t.Errorf("profile = (%s,%d), want (%s,300)", gotProfileType, gotTotal, profiling.TypeCPUNanos)+ }+}diff --git a/backend/app/repositories/testhelper_sqlite_test.go b/backend/app/repositories/testhelper_sqlite_test.goindex 5ea67b7a..8b75604a 100644--- a/backend/app/repositories/testhelper_sqlite_test.go+++ b/backend/app/repositories/testhelper_sqlite_test.go@@ -203,6 +203,51 @@ CREATE TABLE IF NOT EXISTS fired_notifications (url TEXT NOT NULL DEFAULT '',fired_at DATETIME NOT NULL);++CREATE TABLE IF NOT EXISTS profiling_stacks (+ project_id TEXT NOT NULL,+ service_name TEXT NOT NULL DEFAULT '',+ stack_hash INTEGER NOT NULL,+ stack TEXT NOT NULL DEFAULT '[]',+ last_seen DATETIME NOT NULL,+ UNIQUE(project_id, service_name, stack_hash)+);++CREATE TABLE IF NOT EXISTS profiling_samples (+ project_id TEXT NOT NULL,+ profile_id TEXT NOT NULL,+ service_name TEXT NOT NULL DEFAULT '',+ type TEXT NOT NULL DEFAULT '',+ start_time DATETIME NOT NULL,+ end_time DATETIME NOT NULL,+ stack_hash INTEGER NOT NULL,+ value INTEGER NOT NULL DEFAULT 0,+ labels TEXT NOT NULL DEFAULT '{}',+ server_name TEXT NOT NULL DEFAULT '',+ app_version TEXT NOT NULL DEFAULT '',+ trace_id TEXT NOT NULL DEFAULT '',+ span_id TEXT NOT NULL DEFAULT ''+);+CREATE INDEX IF NOT EXISTS idx_profiling_samples_query ON profiling_samples(project_id, type, service_name, start_time);++CREATE TABLE IF NOT EXISTS profiles (+ id TEXT NOT NULL,+ project_id TEXT NOT NULL,+ recorded_at DATETIME NOT NULL,+ duration INTEGER NOT NULL DEFAULT 0,+ service_name TEXT NOT NULL DEFAULT '',+ profile_type TEXT NOT NULL DEFAULT '',+ sample_count INTEGER NOT NULL DEFAULT 0,+ total_value INTEGER NOT NULL DEFAULT 0,+ server_name TEXT NOT NULL DEFAULT '',+ app_version TEXT NOT NULL DEFAULT '',+ attributes TEXT NOT NULL DEFAULT '{}',+ storage_key TEXT NOT NULL DEFAULT '',+ trace_id TEXT NOT NULL DEFAULT '',+ span_id TEXT NOT NULL DEFAULT '',+ distributed_trace_id TEXT DEFAULT NULL+);+CREATE INDEX IF NOT EXISTS idx_profiles_project_recorded ON profiles(project_id, recorded_at);`statements := splitStatements(ddl)diff --git a/backend/go.mod b/backend/go.modindex 5f2313e3..cb48690a 100644--- a/backend/go.mod+++ b/backend/go.mod@@ -11,6 +11,7 @@ require (github.com/coreos/go-systemd/v22 v22.6.0github.com/dop251/goja v0.0.0-20260607120635-348e6bea910dgithub.com/golang-jwt/jwt/v5 v5.3.0+ github.com/google/pprof v0.0.0-20260604005048-7023385849c0github.com/gorilla/securecookie v1.1.2github.com/gorilla/sessions v1.4.0github.com/lib/pq v1.10.9diff --git a/backend/go.sum b/backend/go.sumindex e4cede5a..de86820f 100644--- a/backend/go.sum+++ b/backend/go.sum@@ -179,6 +179,8 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=+github.com/google/pprof v0.0.0-20260604005048-7023385849c0 h1:h1QTMDl6q9wDvDCJVpKQSjgleGFYnd2fOxmg2K+6BGE=+github.com/google/pprof v0.0.0-20260604005048-7023385849c0/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=