The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
fix(profiling): reconcile duplicate BuildRows from parallel PRs (#231)
FrameAutomatad8580db48baf
diff --git a/backend/app/controllers/clientcontrollers/profile_ingest.controller.go b/backend/app/controllers/clientcontrollers/profile_ingest.controller.goindex d62269b8..d85f35e5 100644--- a/backend/app/controllers/clientcontrollers/profile_ingest.controller.go+++ b/backend/app/controllers/clientcontrollers/profile_ingest.controller.go@@ -68,15 +68,7 @@ func (e profileIngestController) Ingest(c *gin.Context) {}convertStart := time.Now()- var stacks []models.ProfileStack- var samples []models.ProfileSample- var profiles []models.Profile- for _, d := range decoded {- s, sm, p := profiling.BuildRows(d, projectId)- stacks = append(stacks, s...)- samples = append(samples, sm...)- profiles = append(profiles, p...)- }+ stacks, samples, profiles := profiling.BuildRows(projectId, decoded)convertMs := float64(time.Since(convertStart).Microseconds()) / 1000.0insertStart := time.Now()diff --git a/backend/app/profiling/build_rows.go b/backend/app/profiling/build_rows.godeleted file mode 100644index 5be56341..00000000--- a/backend/app/profiling/build_rows.go+++ /dev/null@@ -1,74 +0,0 @@-package profiling--import (- "github.com/google/uuid"- "github.com/tracewayapp/traceway/backend/app/models"-)--func BuildRows(d Decoded, projectId uuid.UUID) ([]models.ProfileStack, []models.ProfileSample, []models.Profile) {- stacks := make([]models.ProfileStack, 0, len(d.Stacks))- for _, s := range d.Stacks {- stacks = append(stacks, models.ProfileStack{- ProjectId: projectId,- ServiceName: d.Meta.ServiceName,- StackHash: s.Hash,- Stack: s.Frames,- LastSeen: d.Meta.End,- })- }-- traceId, spanId := "", ""- if d.Meta.TraceId != nil {- traceId = *d.Meta.TraceId- }- if d.Meta.SpanId != nil {- spanId = *d.Meta.SpanId- }-- profileByType := map[string]*models.Profile{}- var typeOrder []string- samples := make([]models.ProfileSample, 0, len(d.Samples))- for _, s := range d.Samples {- p := profileByType[s.Type]- if p == nil {- p = &models.Profile{- Id: uuid.New(),- ProjectId: projectId,- RecordedAt: d.Meta.Start,- Duration: d.Meta.End.Sub(d.Meta.Start),- ServiceName: d.Meta.ServiceName,- ProfileType: s.Type,- ServerName: d.Meta.ServerName,- AppVersion: d.Meta.AppVersion,- Attributes: d.Meta.Attributes,- TraceId: traceId,- SpanId: spanId,- }- profileByType[s.Type] = p- typeOrder = append(typeOrder, s.Type)- }- p.SampleCount++- p.TotalValue += s.Value- samples = append(samples, models.ProfileSample{- ProjectId: projectId,- ProfileId: p.Id,- ServiceName: d.Meta.ServiceName,- Type: s.Type,- Start: d.Meta.Start,- End: d.Meta.End,- StackHash: s.StackHash,- Value: s.Value,- ServerName: d.Meta.ServerName,- AppVersion: d.Meta.AppVersion,- TraceId: traceId,- SpanId: spanId,- })- }-- profiles := make([]models.Profile, 0, len(typeOrder))- for _, t := range typeOrder {- profiles = append(profiles, *profileByType[t])- }-- return stacks, samples, profiles-}diff --git a/backend/app/profiling/build_rows_test.go b/backend/app/profiling/build_rows_test.godeleted file mode 100644index 43cebb81..00000000--- a/backend/app/profiling/build_rows_test.go+++ /dev/null@@ -1,130 +0,0 @@-package profiling--import (- "testing"- "time"-- "github.com/google/uuid"- "github.com/tracewayapp/traceway/backend/app/models"-)--func TestBuildRows_PerTypeProfileRows(t *testing.T) {- projectId := uuid.MustParse("00000000-0000-0000-0000-000000000007")- start := time.Unix(1_700_000_000, 0).UTC()- d := Decoded{- Meta: Meta{- ServiceName: "checkout",- Start: start,- End: start.Add(30 * time.Second),- ServerName: "pod-a",- AppVersion: "1.2.3",- },- Stacks: []Stack{- {Hash: 0xAA, Frames: []string{"main.main", "main.work"}},- },- Samples: []Sample{- {Type: TypeHeapAllocSpace, StackHash: 0xAA, Value: 4096},- {Type: TypeHeapInuseSpace, StackHash: 0xAA, Value: 2048},- },- }-- stacks, samples, profiles := BuildRows(d, projectId)-- if len(stacks) != 1 {- t.Fatalf("expected 1 stack, got %d", len(stacks))- }- if stacks[0].ProjectId != projectId || stacks[0].ServiceName != "checkout" || stacks[0].StackHash != 0xAA {- t.Errorf("stack identity wrong: %+v", stacks[0])- }- if !stacks[0].LastSeen.Equal(d.Meta.End) {- t.Errorf("stack LastSeen = %v, want profile End %v", stacks[0].LastSeen, d.Meta.End)- }-- if len(profiles) != 2 {- t.Fatalf("expected one profile row per type (alloc + inuse), got %d", len(profiles))- }- byType := map[string]models.Profile{}- for _, p := range profiles {- byType[p.ProfileType] = p- if p.Id == uuid.Nil {- t.Errorf("profile %q has nil id", p.ProfileType)- }- if p.ProjectId != projectId || p.ServiceName != "checkout" {- t.Errorf("profile identity wrong: %+v", p)- }- if p.ServerName != "pod-a" || p.AppVersion != "1.2.3" {- t.Errorf("profile provenance not carried: %+v", p)- }- if !p.RecordedAt.Equal(start) {- t.Errorf("profile RecordedAt = %v, want %v", p.RecordedAt, start)- }- if p.Duration != 30*time.Second {- t.Errorf("profile Duration = %v, want 30s", p.Duration)- }- if p.SampleCount != 1 {- t.Errorf("profile %q SampleCount = %d, want 1", p.ProfileType, p.SampleCount)- }- }- if alloc, ok := byType[TypeHeapAllocSpace]; !ok || alloc.TotalValue != 4096 {- t.Errorf("alloc profile TotalValue = %d, want 4096", alloc.TotalValue)- }- if inuse, ok := byType[TypeHeapInuseSpace]; !ok || inuse.TotalValue != 2048 {- t.Errorf("inuse profile TotalValue = %d, want 2048", inuse.TotalValue)- }- if byType[TypeHeapAllocSpace].Id == byType[TypeHeapInuseSpace].Id {- t.Error("per-type profiles must have distinct ids")- }-- if len(samples) != 2 {- t.Fatalf("expected 2 samples, got %d", len(samples))- }- for _, s := range samples {- want := byType[s.Type]- if s.ProfileId != want.Id {- t.Errorf("sample type %q ProfileId = %v, want matching profile id %v", s.Type, s.ProfileId, want.Id)- }- if s.ProjectId != projectId || s.ServiceName != "checkout" || s.StackHash != 0xAA {- t.Errorf("sample identity wrong: %+v", s)- }- if !s.Start.Equal(start) || !s.End.Equal(start.Add(30*time.Second)) {- t.Errorf("sample window wrong: %+v", s)- }- if s.ServerName != "pod-a" || s.AppVersion != "1.2.3" {- t.Errorf("sample provenance not carried: %+v", s)- }- }-}--func TestBuildRows_CPUSingleType(t *testing.T) {- projectId := uuid.New()- start := time.Unix(1_700_000_500, 0).UTC()- d := Decoded{- Meta: Meta{ServiceName: "api", Start: start, End: start.Add(10 * time.Second)},- Stacks: []Stack{{Hash: 1, Frames: []string{"main.main"}}, {Hash: 2, Frames: []string{"main.main", "main.idle"}}},- Samples: []Sample{- {Type: TypeCPUNanos, StackHash: 1, Value: 300},- {Type: TypeCPUNanos, StackHash: 2, Value: 100},- },- }-- _, samples, profiles := BuildRows(d, projectId)-- if len(profiles) != 1 {- t.Fatalf("expected 1 cpu profile, got %d", len(profiles))- }- p := profiles[0]- if p.ProfileType != TypeCPUNanos {- t.Errorf("ProfileType = %q, want %q", p.ProfileType, TypeCPUNanos)- }- if p.TotalValue != 400 {- t.Errorf("TotalValue = %d, want 400 (300+100)", p.TotalValue)- }- if p.SampleCount != 2 {- t.Errorf("SampleCount = %d, want 2", p.SampleCount)- }- for _, s := range samples {- if s.ProfileId != p.Id {- t.Errorf("sample ProfileId = %v, want %v", s.ProfileId, p.Id)- }- }-}diff --git a/backend/app/profiling/rows.go b/backend/app/profiling/rows.goindex 4d522082..f488ee7c 100644--- a/backend/app/profiling/rows.go+++ b/backend/app/profiling/rows.go@@ -26,42 +26,56 @@ func BuildRows(projectId uuid.UUID, decoded []Decoded) ([]models.ProfileStack, [})}- byType := make(map[string][]Sample)- for _, s := range d.Samples {- byType[s.Type] = append(byType[s.Type], s)+ traceId, spanId := "", ""+ if d.Meta.TraceId != nil {+ traceId = *d.Meta.TraceId+ }+ if d.Meta.SpanId != nil {+ spanId = *d.Meta.SpanId}- for typ, group := range byType {- profileId := uuid.New()- var total int64- for _, s := range group {- total += s.Value- samples = append(samples, models.ProfileSample{+ profileByType := map[string]*models.Profile{}+ var typeOrder []string+ for _, s := range d.Samples {+ p := profileByType[s.Type]+ if p == nil {+ p = &models.Profile{+ Id: uuid.New(),ProjectId: projectId,- ProfileId: profileId,+ RecordedAt: d.Meta.Start,+ Duration: d.Meta.End.Sub(d.Meta.Start),ServiceName: d.Meta.ServiceName,- Type: typ,- Start: d.Meta.Start,- End: d.Meta.End,- StackHash: s.StackHash,- Value: s.Value,+ ProfileType: s.Type,ServerName: d.Meta.ServerName,AppVersion: d.Meta.AppVersion,- })+ Attributes: d.Meta.Attributes,+ TraceId: traceId,+ SpanId: spanId,+ }+ profileByType[s.Type] = p+ typeOrder = append(typeOrder, s.Type)}- profiles = append(profiles, models.Profile{- Id: profileId,+ p.SampleCount+++ p.TotalValue += s.Value+ samples = append(samples, models.ProfileSample{ProjectId: projectId,- RecordedAt: d.Meta.Start,- Duration: d.Meta.End.Sub(d.Meta.Start),+ ProfileId: p.Id,ServiceName: d.Meta.ServiceName,- ProfileType: typ,- SampleCount: uint64(len(group)),- TotalValue: total,+ Type: s.Type,+ Start: d.Meta.Start,+ End: d.Meta.End,+ StackHash: s.StackHash,+ Value: s.Value,ServerName: d.Meta.ServerName,AppVersion: d.Meta.AppVersion,+ TraceId: traceId,+ SpanId: spanId,})}++ for _, t := range typeOrder {+ profiles = append(profiles, *profileByType[t])+ }}return stacks, samples, profilesdiff --git a/backend/app/profiling/rows_test.go b/backend/app/profiling/rows_test.goindex e953cc93..ab557ac4 100644--- a/backend/app/profiling/rows_test.go+++ b/backend/app/profiling/rows_test.go@@ -134,3 +134,92 @@ func TestBuildRows_MultiTypeSplitsProfiles(t *testing.T) {}}}++func TestBuildRows_PerTypeProfileRowsCarryProvenance(t *testing.T) {+ projectId := uuid.MustParse("00000000-0000-0000-0000-000000000007")+ start := time.Unix(1_700_000_000, 0).UTC()+ d := Decoded{+ Meta: Meta{+ ServiceName: "checkout",+ Start: start,+ End: start.Add(30 * time.Second),+ ServerName: "pod-a",+ AppVersion: "1.2.3",+ },+ Stacks: []Stack{+ {Hash: 0xAA, Frames: []string{"main.main", "main.work"}},+ },+ Samples: []Sample{+ {Type: TypeHeapAllocSpace, StackHash: 0xAA, Value: 4096},+ {Type: TypeHeapInuseSpace, StackHash: 0xAA, Value: 2048},+ },+ }++ stacks, samples, profiles := BuildRows(projectId, []Decoded{d})++ if len(stacks) != 1 || stacks[0].StackHash != 0xAA {+ t.Fatalf("stacks wrong: %+v", stacks)+ }++ if len(profiles) != 2 {+ t.Fatalf("expected one profile row per type (alloc + inuse), got %d", len(profiles))+ }+ byType := map[string]models.Profile{}+ for _, p := range profiles {+ byType[p.ProfileType] = p+ if p.Id == uuid.Nil || p.ProjectId != projectId || p.ServiceName != "checkout" {+ t.Errorf("profile identity wrong: %+v", p)+ }+ if p.ServerName != "pod-a" || p.AppVersion != "1.2.3" {+ t.Errorf("profile provenance not carried: %+v", p)+ }+ if p.SampleCount != 1 {+ t.Errorf("profile %q SampleCount = %d, want 1", p.ProfileType, p.SampleCount)+ }+ }+ if byType[TypeHeapAllocSpace].TotalValue != 4096 || byType[TypeHeapInuseSpace].TotalValue != 2048 {+ t.Errorf("totals wrong: alloc=%d inuse=%d", byType[TypeHeapAllocSpace].TotalValue, byType[TypeHeapInuseSpace].TotalValue)+ }+ if byType[TypeHeapAllocSpace].Id == byType[TypeHeapInuseSpace].Id {+ t.Error("per-type profiles must have distinct ids")+ }++ for _, s := range samples {+ if s.ProfileId != byType[s.Type].Id {+ t.Errorf("sample type %q ProfileId = %v, want %v", s.Type, s.ProfileId, byType[s.Type].Id)+ }+ if s.ServerName != "pod-a" || s.AppVersion != "1.2.3" {+ t.Errorf("sample provenance not carried: %+v", s)+ }+ }+}++func TestBuildRows_MergesAcrossDecodedBatch(t *testing.T) {+ projectId := uuid.New()+ start := time.Unix(1_700_000_500, 0).UTC()+ d1 := Decoded{+ Meta: Meta{ServiceName: "api", Start: start, End: start.Add(10 * time.Second)},+ Stacks: []Stack{{Hash: 1, Frames: []string{"main.main"}}},+ Samples: []Sample{{Type: TypeCPUNanos, StackHash: 1, Value: 300}},+ }+ d2 := Decoded{+ Meta: Meta{ServiceName: "api", Start: start, End: start.Add(10 * time.Second)},+ Stacks: []Stack{{Hash: 2, Frames: []string{"main.main", "main.idle"}}},+ Samples: []Sample{{Type: TypeCPUNanos, StackHash: 2, Value: 100}},+ }++ stacks, samples, profiles := BuildRows(projectId, []Decoded{d1, d2})++ if len(stacks) != 2 {+ t.Fatalf("stacks = %d, want 2 across the batch", len(stacks))+ }+ if len(samples) != 2 {+ t.Fatalf("samples = %d, want 2 across the batch", len(samples))+ }+ if len(profiles) != 2 {+ t.Fatalf("profiles = %d, want 2 (one per decoded, each its own profile id)", len(profiles))+ }+ if profiles[0].Id == profiles[1].Id {+ t.Error("profiles from distinct decoded inputs must have distinct ids")+ }+}