The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
Added fixture tests for OTEL + integrating node otel better
ddux6c328bdbf5f4
diff --git a/README.md b/README.mdindex a498208e..8dfea721 100644--- a/README.md+++ b/README.md@@ -124,6 +124,12 @@ cd backend && go test -v -count=1 ./app/repositories/# ClickHouse + PostgreSQL tests (requires Docker)./scripts/test-backend-pgch.sh++# OTEL trace converter tests (no DB required)+cd backend && go test -v -count=1 ./app/controllers/otelcontrollers/++# Update OTEL golden files after intentional converter changes+cd backend && go test -v -count=1 -args -update ./app/controllers/otelcontrollers/```## Documentationdiff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.goindex 352819fc..59160a9f 100644--- a/backend/app/controllers/otelcontrollers/otel.controller.go+++ b/backend/app/controllers/otelcontrollers/otel.controller.go@@ -3,6 +3,7 @@ package otelcontrollersimport ("context""fmt"+ "log""net/http""github.com/gin-gonic/gin"@@ -13,6 +14,7 @@ import ("github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage"traceway "go.tracewayapp.com"+ "google.golang.org/protobuf/encoding/protojson")type otelController struct{}@@ -20,11 +22,13 @@ type otelController struct{}var OtelController = otelController{}func (o otelController) ExportTraces(c *gin.Context) {+ fmt.Println("A00")projectId, err := middleware.GetProjectId(c)if err != nil {c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))return}+ fmt.Println("A0")if project, exists := c.Get(middleware.ProjectContextKey); exists {if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {if !hooks.CanReport(*p.OrganizationId) {@@ -33,6 +37,7 @@ func (o otelController) ExportTraces(c *gin.Context) {}}}+ fmt.Println("A1")req, err := decodeTraceRequest(c)if err != nil {@@ -40,6 +45,11 @@ func (o otelController) ExportTraces(c *gin.Context) {return}+ fmt.Println("WTF??")+ if jsonBytes, err := protojson.Marshal(req); err == nil {+ log.Printf("[OTEL TRACES] Payload: %s", string(jsonBytes))+ }+endpoints, tasks, spans, exceptions, aiTraces, aiConversations := convertTraces(projectId, req)if len(endpoints) > 0 {@@ -89,14 +99,6 @@ func (o otelController) ExportTraces(c *gin.Context) {exceptionHashes = append(exceptionHashes, ex.ExceptionHash)}- var aiTraceInfos []hooks.AiTraceInfo- for _, at := range aiTraces {- aiTraceInfos = append(aiTraceInfos, hooks.AiTraceInfo{- TraceName: at.TraceName,- TotalCost: at.TotalCost,- })- }-if project, exists := c.Get(middleware.ProjectContextKey); exists {if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {hooks.BroadcastReport(hooks.ReportEvent{@@ -106,7 +108,6 @@ func (o otelController) ExportTraces(c *gin.Context) {ErrorCount: len(exceptions),TaskCount: len(tasks),ExceptionHashes: exceptionHashes,- AiTraces: aiTraceInfos,})}}diff --git a/backend/app/controllers/otelcontrollers/testdata/node_better_auth.json b/backend/app/controllers/otelcontrollers/testdata/node_better_auth.jsonnew file mode 100644index 00000000..7fa268d4--- /dev/null+++ b/backend/app/controllers/otelcontrollers/testdata/node_better_auth.json@@ -0,0 +1,99 @@+{+ "resourceSpans": [+ {+ "resource": {+ "attributes": [+ {"key": "service.name", "value": {"stringValue": "panelshot"}},+ {"key": "service.version", "value": {"stringValue": "0.1.0"}}+ ]+ },+ "scopeSpans": [+ {+ "scope": {"name": "@opentelemetry/instrumentation-net", "version": "0.58.0"},+ "spans": [+ {+ "traceId": "59e33ba274805c8ac4d3c3d3255f4b2a",+ "spanId": "fab4307b2bfdc856",+ "parentSpanId": "3d30681bfa029543",+ "flags": 257,+ "name": "tcp.connect",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995349118000000",+ "endTimeUnixNano": "1774995349122471125",+ "attributes": [+ {"key": "net.transport", "value": {"stringValue": "ip_tcp"}},+ {"key": "net.peer.name", "value": {"stringValue": "postgres"}},+ {"key": "net.peer.port", "value": {"intValue": "5432"}}+ ],+ "status": {}+ }+ ]+ },+ {+ "scope": {"name": "better-auth", "version": "1.5.6"},+ "spans": [+ {+ "traceId": "59e33ba274805c8ac4d3c3d3255f4b2a",+ "spanId": "3d30681bfa029543",+ "parentSpanId": "9c31a2fb43bec7f5",+ "flags": 257,+ "name": "db findOne session",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995349113000000",+ "endTimeUnixNano": "1774995349133810500",+ "attributes": [+ {"key": "db.operation.name", "value": {"stringValue": "findOne"}},+ {"key": "db.collection.name", "value": {"stringValue": "session"}}+ ],+ "status": {}+ },+ {+ "traceId": "59e33ba274805c8ac4d3c3d3255f4b2a",+ "spanId": "e795832cb5a0e841",+ "parentSpanId": "9c31a2fb43bec7f5",+ "flags": 257,+ "name": "db findOne user",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995349134000000",+ "endTimeUnixNano": "1774995349135635750",+ "attributes": [+ {"key": "db.operation.name", "value": {"stringValue": "findOne"}},+ {"key": "db.collection.name", "value": {"stringValue": "user"}}+ ],+ "status": {}+ },+ {+ "traceId": "59e33ba274805c8ac4d3c3d3255f4b2a",+ "spanId": "9c31a2fb43bec7f5",+ "parentSpanId": "e0fa18d76871a9d5",+ "flags": 257,+ "name": "handler /get-session",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995349107000000",+ "endTimeUnixNano": "1774995349135923958",+ "attributes": [+ {"key": "http.route", "value": {"stringValue": "/get-session"}},+ {"key": "better_auth.operation_id", "value": {"stringValue": "getSession"}}+ ],+ "status": {}+ },+ {+ "traceId": "59e33ba274805c8ac4d3c3d3255f4b2a",+ "spanId": "e0fa18d76871a9d5",+ "flags": 257,+ "name": "GET /get-session",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995349106000000",+ "endTimeUnixNano": "1774995349135902042",+ "attributes": [+ {"key": "http.route", "value": {"stringValue": "/get-session"}},+ {"key": "better_auth.operation_id", "value": {"stringValue": "getSession"}}+ ],+ "status": {}+ }+ ]+ }+ ]+ }+ ]+}diff --git a/backend/app/controllers/otelcontrollers/testdata/node_better_auth.json.golden.json b/backend/app/controllers/otelcontrollers/testdata/node_better_auth.json.golden.jsonnew file mode 100644index 00000000..a8951622--- /dev/null+++ b/backend/app/controllers/otelcontrollers/testdata/node_better_auth.json.golden.json@@ -0,0 +1,37 @@+{+ "endpointCount": 1,+ "endpoints": [+ {+ "endpoint": "GET /get-session",+ "statusCode": 0,+ "serverName": "panelshot",+ "appVersion": "0.1.0"+ }+ ],+ "taskCount": 0,+ "spanCount": 4,+ "spans": [+ {+ "name": "db findOne session",+ "linkedToParent": true+ },+ {+ "name": "db findOne user",+ "linkedToParent": true+ },+ {+ "name": "handler /get-session",+ "linkedToParent": true+ },+ {+ "name": "tcp.connect",+ "linkedToParent": true+ }+ ],+ "exceptionCount": 0,+ "exceptions": [],+ "aiTraceCount": 0,+ "aiTraces": [],+ "conversationCount": 0,+ "allSpansLinked": true+}\ No newline at end of filediff --git a/backend/app/controllers/otelcontrollers/testdata/node_sign_in.json b/backend/app/controllers/otelcontrollers/testdata/node_sign_in.jsonnew file mode 100644index 00000000..ec511d59--- /dev/null+++ b/backend/app/controllers/otelcontrollers/testdata/node_sign_in.json@@ -0,0 +1,127 @@+{+ "resourceSpans": [+ {+ "resource": {+ "attributes": [+ {"key": "service.name", "value": {"stringValue": "panelshot"}},+ {"key": "service.version", "value": {"stringValue": "0.1.0"}}+ ]+ },+ "scopeSpans": [+ {+ "scope": {"name": "@opentelemetry/instrumentation-net", "version": "0.58.0"},+ "spans": [+ {+ "traceId": "80f4fa0ed0d7273a63572540b8053ad9",+ "spanId": "d3091afa9e646793",+ "parentSpanId": "2ac07369848d8539",+ "flags": 257,+ "name": "tcp.connect",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995633007000000",+ "endTimeUnixNano": "1774995633008332125",+ "attributes": [+ {"key": "net.transport", "value": {"stringValue": "ip_tcp"}},+ {"key": "net.peer.name", "value": {"stringValue": "postgres"}},+ {"key": "net.peer.port", "value": {"intValue": "5432"}}+ ],+ "status": {}+ }+ ]+ },+ {+ "scope": {"name": "better-auth", "version": "1.5.6"},+ "spans": [+ {+ "traceId": "80f4fa0ed0d7273a63572540b8053ad9",+ "spanId": "2ac07369848d8539",+ "parentSpanId": "aca958ef32ba7c4c",+ "flags": 257,+ "name": "db findOne user",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995633005000000",+ "endTimeUnixNano": "1774995633021000333",+ "attributes": [+ {"key": "db.operation.name", "value": {"stringValue": "findOne"}},+ {"key": "db.collection.name", "value": {"stringValue": "user"}}+ ],+ "status": {}+ },+ {+ "traceId": "80f4fa0ed0d7273a63572540b8053ad9",+ "spanId": "c26fc11ea2e2d7fd",+ "parentSpanId": "aca958ef32ba7c4c",+ "flags": 257,+ "name": "db findMany account",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995633022000000",+ "endTimeUnixNano": "1774995633023763458",+ "attributes": [+ {"key": "db.operation.name", "value": {"stringValue": "findMany"}},+ {"key": "db.collection.name", "value": {"stringValue": "account"}}+ ],+ "status": {}+ },+ {+ "traceId": "80f4fa0ed0d7273a63572540b8053ad9",+ "spanId": "aca958ef32ba7c4c",+ "parentSpanId": "4aac2608cd549092",+ "flags": 257,+ "name": "handler /sign-in/email",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995633003000000",+ "endTimeUnixNano": "1774995633178519542",+ "attributes": [+ {"key": "http.route", "value": {"stringValue": "/sign-in/email"}},+ {"key": "better_auth.operation_id", "value": {"stringValue": "signInEmail"}}+ ],+ "events": [+ {+ "timeUnixNano": "1774995633178497834",+ "name": "exception",+ "attributes": [+ {"key": "exception.type", "value": {"stringValue": "APIError"}},+ {"key": "exception.message", "value": {"stringValue": "Invalid email or password"}},+ {"key": "exception.stacktrace", "value": {"stringValue": "APIError: Invalid email or password"}}+ ]+ }+ ],+ "status": {"message": "Invalid email or password", "code": "STATUS_CODE_ERROR"}+ },+ {+ "traceId": "80f4fa0ed0d7273a63572540b8053ad9",+ "spanId": "37b95aae03f5332d",+ "parentSpanId": "4aac2608cd549092",+ "flags": 257,+ "name": "hook after /sign-in/email plugin:bearer",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995633179000000",+ "endTimeUnixNano": "1774995633179168291",+ "attributes": [+ {"key": "better_auth.hook.type", "value": {"stringValue": "after"}},+ {"key": "http.route", "value": {"stringValue": "/sign-in/email"}},+ {"key": "better_auth.context", "value": {"stringValue": "plugin:bearer"}},+ {"key": "better_auth.operation_id", "value": {"stringValue": "signInEmail"}}+ ],+ "status": {}+ },+ {+ "traceId": "80f4fa0ed0d7273a63572540b8053ad9",+ "spanId": "4aac2608cd549092",+ "flags": 257,+ "name": "POST /sign-in/email",+ "kind": "SPAN_KIND_INTERNAL",+ "startTimeUnixNano": "1774995633002000000",+ "endTimeUnixNano": "1774995633178560208",+ "attributes": [+ {"key": "http.route", "value": {"stringValue": "/sign-in/email"}},+ {"key": "better_auth.operation_id", "value": {"stringValue": "signInEmail"}}+ ],+ "status": {}+ }+ ]+ }+ ]+ }+ ]+}diff --git a/backend/app/controllers/otelcontrollers/testdata/node_sign_in.json.golden.json b/backend/app/controllers/otelcontrollers/testdata/node_sign_in.json.golden.jsonnew file mode 100644index 00000000..931f6439--- /dev/null+++ b/backend/app/controllers/otelcontrollers/testdata/node_sign_in.json.golden.json@@ -0,0 +1,46 @@+{+ "endpointCount": 1,+ "endpoints": [+ {+ "endpoint": "POST /sign-in/email",+ "statusCode": 0,+ "serverName": "panelshot",+ "appVersion": "0.1.0"+ }+ ],+ "taskCount": 0,+ "spanCount": 5,+ "spans": [+ {+ "name": "db findMany account",+ "linkedToParent": true+ },+ {+ "name": "db findOne user",+ "linkedToParent": true+ },+ {+ "name": "handler /sign-in/email",+ "linkedToParent": true+ },+ {+ "name": "hook after /sign-in/email plugin:bearer",+ "linkedToParent": true+ },+ {+ "name": "tcp.connect",+ "linkedToParent": true+ }+ ],+ "exceptionCount": 1,+ "exceptions": [+ {+ "stackTrace": "APIError: Invalid email or password\nAPIError: Invalid email or password",+ "traceType": "task"+ }+ ],+ "aiTraceCount": 0,+ "aiTraces": [],+ "conversationCount": 0,+ "allSpansLinked": true+}\ No newline at end of filediff --git a/backend/app/controllers/otelcontrollers/testdata/openrouter_ai_trace.json b/backend/app/controllers/otelcontrollers/testdata/openrouter_ai_trace.jsonnew file mode 100644index 00000000..1a3c211a--- /dev/null+++ b/backend/app/controllers/otelcontrollers/testdata/openrouter_ai_trace.json@@ -0,0 +1,50 @@+{+ "resourceSpans": [+ {+ "resource": {+ "attributes": [+ {"key": "service.name", "value": {"stringValue": "openrouter"}},+ {"key": "openrouter.trace.id", "value": {"stringValue": "5449f21c-da7b-4c6f-9b68-47c76347fd43"}},+ {"key": "service.version", "value": {"stringValue": "1.0.0"}}+ ]+ },+ "scopeSpans": [+ {+ "scope": {"name": "openrouter"},+ "spans": [+ {+ "attributes": [+ {"key": "trace.name", "value": {"stringValue": "Test Trace - OpenRouter Observability"}},+ {"key": "user.id", "value": {"stringValue": "org_3BZjaeIcd6CYxEvJbTKUILmLA67"}},+ {"key": "gen_ai.operation.name", "value": {"stringValue": "chat"}},+ {"key": "gen_ai.system", "value": {"stringValue": "openai"}},+ {"key": "gen_ai.request.model", "value": {"stringValue": "openai/gpt-4-turbo"}},+ {"key": "gen_ai.response.model", "value": {"stringValue": "openai/gpt-4-turbo"}},+ {"key": "gen_ai.usage.input_tokens", "value": {"intValue": 50}},+ {"key": "gen_ai.usage.output_tokens", "value": {"intValue": 100}},+ {"key": "gen_ai.usage.total_tokens", "value": {"intValue": 150}},+ {"key": "gen_ai.usage.input_cost", "value": {"doubleValue": 0.005}},+ {"key": "gen_ai.usage.output_cost", "value": {"doubleValue": 0.015}},+ {"key": "gen_ai.usage.total_cost", "value": {"doubleValue": 0.02}},+ {"key": "gen_ai.request.temperature", "value": {"doubleValue": 0.7}},+ {"key": "gen_ai.request.max_tokens", "value": {"intValue": 150}},+ {"key": "gen_ai.response.finish_reason", "value": {"stringValue": "stop"}},+ {"key": "gen_ai.prompt", "value": {"stringValue": "{\"messages\":[{\"role\":\"user\",\"content\":\"What is the capital of France?\"}]}"}},+ {"key": "gen_ai.completion", "value": {"stringValue": "{\"choices\":[{\"message\":{\"role\":\"assistant\",\"content\":\"The capital of France is Paris.\"}}]}"}},+ {"key": "gen_ai.usage.input_tokens.cached", "value": {"intValue": 20}},+ {"key": "gen_ai.usage.output_tokens.reasoning", "value": {"intValue": 10}}+ ],+ "endTimeUnixNano": "1774926644373000000",+ "kind": 3,+ "name": "Test Generation",+ "spanId": "0806f22e9f134e7e",+ "startTimeUnixNano": "1774926642873000000",+ "status": {"code": 1},+ "traceId": "ef5ad7a3070fecf76bc3180145d97678"+ }+ ]+ }+ ]+ }+ ]+}diff --git a/backend/app/controllers/otelcontrollers/testdata/openrouter_ai_trace.json.golden.json b/backend/app/controllers/otelcontrollers/testdata/openrouter_ai_trace.json.golden.jsonnew file mode 100644index 00000000..44b00cc9--- /dev/null+++ b/backend/app/controllers/otelcontrollers/testdata/openrouter_ai_trace.json.golden.json@@ -0,0 +1,30 @@+{+ "endpointCount": 0,+ "endpoints": [],+ "taskCount": 0,+ "spanCount": 0,+ "spans": [],+ "exceptionCount": 0,+ "exceptions": [],+ "aiTraceCount": 1,+ "aiTraces": [+ {+ "traceName": "Test Trace - OpenRouter Observability",+ "model": "openai/gpt-4-turbo",+ "provider": "openai",+ "operation": "chat",+ "inputTokens": 50,+ "outputTokens": 100,+ "totalTokens": 150,+ "cachedTokens": 20,+ "reasoningTokens": 10,+ "inputCost": 0.005,+ "outputCost": 0.015,+ "totalCost": 0.02,+ "finishReason": "stop",+ "statusCode": 1+ }+ ],+ "conversationCount": 1,+ "allSpansLinked": true+}\ No newline at end of filediff --git a/backend/app/controllers/otelcontrollers/trace_converter.go b/backend/app/controllers/otelcontrollers/trace_converter.goindex ff1dab08..6bfd0307 100644--- a/backend/app/controllers/otelcontrollers/trace_converter.go+++ b/backend/app/controllers/otelcontrollers/trace_converter.go@@ -40,99 +40,123 @@ func convertTraces(projectId uuid.UUID, req *coltracepb.ExportTraceServiceReques}}- spanToTraceId := map[string]uuid.UUID{}+ // First pass: collect all spans and build parent→child relationships+ type spanEntry struct {+ span *tracepb.Span+ scopeName string+ }+ var allSpans []spanEntry+ parentMap := map[string]string{} // childSpanId → parentSpanIdfor _, ss := range rs.ScopeSpans {for _, span := range ss.Spans {- spanAttrs := span.Attributes- allAttrs := extractAttributes(spanAttrs)+ allSpans = append(allSpans, spanEntry{span: span, scopeName: ss.GetScope().GetName()})+ if len(span.ParentSpanId) > 0 {+ parentMap[string(span.SpanId)] = string(span.ParentSpanId)+ }+ }+ }- isRoot := len(span.ParentSpanId) == 0+ // Build spanToTraceId by walking up to the root for each span+ spanToTraceId := map[string]uuid.UUID{}+ var resolveTraceId func(spanIdStr string) uuid.UUID+ resolveTraceId = func(spanIdStr string) uuid.UUID {+ if tid, ok := spanToTraceId[spanIdStr]; ok {+ return tid+ }+ if parentId, hasParent := parentMap[spanIdStr]; hasParent {+ tid := resolveTraceId(parentId)+ spanToTraceId[spanIdStr] = tid+ return tid+ }+ // This is a root span+ tid := uuid.New()+ spanToTraceId[spanIdStr] = tid+ return tid+ }+ for _, entry := range allSpans {+ resolveTraceId(string(entry.span.SpanId))+ }- spanIdStr := string(span.SpanId)- var traceId uuid.UUID+ // Second pass: process spans and create records+ for _, entry := range allSpans {+ span := entry.span+ spanAttrs := span.Attributes+ allAttrs := extractAttributes(spanAttrs)- if isRoot {- traceId = uuid.New()- } else if foundTraceId, ok := spanToTraceId[string(span.ParentSpanId)]; ok {- traceId = foundTraceId- } else {- traceId = uuid.New()- }+ isRoot := len(span.ParentSpanId) == 0+ traceId := spanToTraceId[string(span.SpanId)]- spanToTraceId[spanIdStr] = traceId+ spanId := otelSpanIDToUUID(span.SpanId)+ startTime := nanoToTime(span.StartTimeUnixNano)+ endTime := nanoToTime(span.EndTimeUnixNano)+ duration := endTime.Sub(startTime)- spanId := otelSpanIDToUUID(span.SpanId)- startTime := nanoToTime(span.StartTimeUnixNano)- endTime := nanoToTime(span.EndTimeUnixNano)- duration := endTime.Sub(startTime)-- var distributedTraceId *uuid.UUID- if dtid := getStringAttribute(spanAttrs, "traceway.distributed_trace_id"); dtid != "" {- if parsed, err := uuid.Parse(dtid); err == nil {- distributedTraceId = &parsed- }+ var distributedTraceId *uuid.UUID+ if dtid := getStringAttribute(spanAttrs, "traceway.distributed_trace_id"); dtid != "" {+ if parsed, err := uuid.Parse(dtid); err == nil {+ distributedTraceId = &parsed}+ }- if isRoot {- if span.Kind == tracepb.Span_SPAN_KIND_SERVER && hasHTTPAttributes(spanAttrs) {- ep := buildEndpoint(- traceId, projectId, span, spanAttrs, allAttrs,- startTime, duration, serverName, appVersion,- )- ep.DistributedTraceId = distributedTraceId- endpoints = append(endpoints, ep)- } else if span.Kind == tracepb.Span_SPAN_KIND_CONSUMER {- t := buildTask(- traceId, projectId, span, allAttrs,- startTime, endTime, duration, serverName, appVersion,- )- t.DistributedTraceId = distributedTraceId- tasks = append(tasks, t)- } else if hasGenAiAttributes(spanAttrs) {- aiTrace := buildAiTrace(- traceId, projectId, span, spanAttrs, allAttrs,- startTime, duration, serverName, appVersion,- )- aiTraces = append(aiTraces, aiTrace)- if conv := extractConversation(spanAttrs, projectId, traceId); conv != nil {- aiConversations = append(aiConversations, *conv)- }- } else {- continue+ if isRoot {+ if (span.Kind == tracepb.Span_SPAN_KIND_SERVER || span.Kind == tracepb.Span_SPAN_KIND_INTERNAL) && hasHTTPAttributes(spanAttrs) {+ ep := buildEndpoint(+ traceId, projectId, span, spanAttrs, allAttrs,+ startTime, duration, serverName, appVersion,+ )+ ep.DistributedTraceId = distributedTraceId+ endpoints = append(endpoints, ep)+ } else if span.Kind == tracepb.Span_SPAN_KIND_CONSUMER {+ t := buildTask(+ traceId, projectId, span, allAttrs,+ startTime, endTime, duration, serverName, appVersion,+ )+ t.DistributedTraceId = distributedTraceId+ tasks = append(tasks, t)+ } else if hasGenAiAttributes(spanAttrs) {+ aiTrace := buildAiTrace(+ traceId, projectId, span, spanAttrs, allAttrs,+ startTime, duration, serverName, appVersion,+ )+ aiTraces = append(aiTraces, aiTrace)+ if conv := extractConversation(spanAttrs, projectId, traceId); conv != nil {+ aiConversations = append(aiConversations, *conv)}} else {- spanName := span.Name- if dbQuery := getStringAttribute(spanAttrs, "db.query.text"); dbQuery != "" {- spanName = dbQuery- } else if dbStatement := getStringAttribute(spanAttrs, "db.statement"); dbStatement != "" {- spanName = dbStatement- }-- spans = append(spans, models.Span{- Id: spanId,- TraceId: traceId,- ProjectId: projectId,- Name: spanName,- StartTime: startTime,- Duration: duration,- RecordedAt: startTime,- })+ continue}-- traceType := "task"- if isRoot && span.Kind == tracepb.Span_SPAN_KIND_SERVER && hasHTTPAttributes(spanAttrs) {- traceType = "endpoint"+ } else {+ spanName := span.Name+ if dbQuery := getStringAttribute(spanAttrs, "db.query.text"); dbQuery != "" {+ spanName = dbQuery+ } else if dbStatement := getStringAttribute(spanAttrs, "db.statement"); dbStatement != "" {+ spanName = dbStatement}- for _, event := range span.Events {- if event.Name == "exception" {- exc := buildException(- projectId, traceId, traceType, event,- serverName, appVersion,- )- exc.DistributedTraceId = distributedTraceId- exceptions = append(exceptions, exc)- }+ spans = append(spans, models.Span{+ Id: spanId,+ TraceId: traceId,+ ProjectId: projectId,+ Name: spanName,+ StartTime: startTime,+ Duration: duration,+ RecordedAt: startTime,+ })+ }++ traceType := "task"+ if isRoot && (span.Kind == tracepb.Span_SPAN_KIND_SERVER || span.Kind == tracepb.Span_SPAN_KIND_INTERNAL) && hasHTTPAttributes(spanAttrs) {+ traceType = "endpoint"+ }++ for _, event := range span.Events {+ if event.Name == "exception" {+ exc := buildException(+ projectId, traceId, traceType, event,+ serverName, appVersion,+ )+ exc.DistributedTraceId = distributedTraceId+ exceptions = append(exceptions, exc)}}}diff --git a/backend/app/controllers/otelcontrollers/trace_converter_test.go b/backend/app/controllers/otelcontrollers/trace_converter_test.gonew file mode 100644index 00000000..5d546b56--- /dev/null+++ b/backend/app/controllers/otelcontrollers/trace_converter_test.go@@ -0,0 +1,402 @@+package otelcontrollers++import (+ "encoding/json"+ "flag"+ "os"+ "sort"+ "testing"++ "github.com/google/uuid"+ commonpb "go.opentelemetry.io/proto/otlp/common/v1"+ coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"+ resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"+ "google.golang.org/protobuf/encoding/protojson"+)++var update = flag.Bool("update", false, "update golden files")++var testProjectId = uuid.MustParse("00000000-0000-0000-0000-000000000001")++// --- Snapshot types (stable, deterministic output) ---++type snapshotEndpoint struct {+ Endpoint string `json:"endpoint"`+ StatusCode int16 `json:"statusCode"`+ ServerName string `json:"serverName"`+ AppVersion string `json:"appVersion"`+}++type snapshotSpan struct {+ Name string `json:"name"`+ LinkedToParent bool `json:"linkedToParent"`+}++type snapshotException struct {+ StackTrace string `json:"stackTrace"`+ TraceType string `json:"traceType"`+}++type snapshotAiTrace struct {+ TraceName string `json:"traceName"`+ Model string `json:"model"`+ Provider string `json:"provider"`+ Operation string `json:"operation"`+ InputTokens int64 `json:"inputTokens"`+ OutputTokens int64 `json:"outputTokens"`+ TotalTokens int64 `json:"totalTokens"`+ CachedTokens int64 `json:"cachedTokens"`+ ReasoningTokens int64 `json:"reasoningTokens"`+ InputCost float64 `json:"inputCost"`+ OutputCost float64 `json:"outputCost"`+ TotalCost float64 `json:"totalCost"`+ FinishReason string `json:"finishReason"`+ StatusCode uint8 `json:"statusCode"`+}++type snapshotResult struct {+ EndpointCount int `json:"endpointCount"`+ Endpoints []snapshotEndpoint `json:"endpoints"`+ TaskCount int `json:"taskCount"`+ SpanCount int `json:"spanCount"`+ Spans []snapshotSpan `json:"spans"`+ ExceptionCount int `json:"exceptionCount"`+ Exceptions []snapshotException `json:"exceptions"`+ AiTraceCount int `json:"aiTraceCount"`+ AiTraces []snapshotAiTrace `json:"aiTraces"`+ ConversationCount int `json:"conversationCount"`+ AllSpansLinked bool `json:"allSpansLinked"`+}++// --- Snapshot tests ---++func TestConvertTraces_Snapshot(t *testing.T) {+ tests := []struct {+ name string+ fixture string+ }{+ {"openrouter_ai_trace", "testdata/openrouter_ai_trace.json"},+ {"node_better_auth", "testdata/node_better_auth.json"},+ {"node_sign_in", "testdata/node_sign_in.json"},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ raw, err := os.ReadFile(tt.fixture)+ if err != nil {+ t.Fatalf("failed to read fixture %s: %v", tt.fixture, err)+ }++ req := &coltracepb.ExportTraceServiceRequest{}+ if err := protojson.Unmarshal(raw, req); err != nil {+ t.Fatalf("failed to unmarshal fixture %s: %v", tt.fixture, err)+ }++ endpoints, _, spans, exceptions, aiTraces, aiConversations := convertTraces(testProjectId, req)++ // Check if all child spans share a trace ID with an endpoint+ endpointIds := map[uuid.UUID]bool{}+ for _, ep := range endpoints {+ endpointIds[ep.Id] = true+ }+ for _, at := range aiTraces {+ endpointIds[at.Id] = true+ }+ allLinked := true+ snapSpans := make([]snapshotSpan, len(spans))+ for i, s := range spans {+ linked := endpointIds[s.TraceId]+ snapSpans[i] = snapshotSpan{Name: s.Name, LinkedToParent: linked}+ if !linked {+ allLinked = false+ }+ }+ sort.Slice(snapSpans, func(i, j int) bool { return snapSpans[i].Name < snapSpans[j].Name })++ snapEndpoints := make([]snapshotEndpoint, len(endpoints))+ for i, ep := range endpoints {+ snapEndpoints[i] = snapshotEndpoint{+ Endpoint: ep.Endpoint,+ StatusCode: ep.StatusCode,+ ServerName: ep.ServerName,+ AppVersion: ep.AppVersion,+ }+ }++ snapExceptions := make([]snapshotException, len(exceptions))+ for i, ex := range exceptions {+ snapExceptions[i] = snapshotException{+ StackTrace: ex.StackTrace,+ TraceType: ex.TraceType,+ }+ }++ snapAiTraces := make([]snapshotAiTrace, len(aiTraces))+ for i, at := range aiTraces {+ snapAiTraces[i] = snapshotAiTrace{+ TraceName: at.TraceName,+ Model: at.Model,+ Provider: at.Provider,+ Operation: at.Operation,+ InputTokens: at.InputTokens,+ OutputTokens: at.OutputTokens,+ TotalTokens: at.TotalTokens,+ CachedTokens: at.CachedTokens,+ ReasoningTokens: at.ReasoningTokens,+ InputCost: at.InputCost,+ OutputCost: at.OutputCost,+ TotalCost: at.TotalCost,+ FinishReason: at.FinishReason,+ StatusCode: at.StatusCode,+ }+ }++ result := snapshotResult{+ EndpointCount: len(endpoints),+ Endpoints: snapEndpoints,+ TaskCount: 0,+ SpanCount: len(spans),+ Spans: snapSpans,+ ExceptionCount: len(exceptions),+ Exceptions: snapExceptions,+ AiTraceCount: len(aiTraces),+ AiTraces: snapAiTraces,+ ConversationCount: len(aiConversations),+ AllSpansLinked: allLinked,+ }++ got, err := json.MarshalIndent(result, "", " ")+ if err != nil {+ t.Fatalf("failed to marshal result: %v", err)+ }++ golden := tt.fixture + ".golden.json"+ if *update {+ if err := os.WriteFile(golden, got, 0644); err != nil {+ t.Fatalf("failed to write golden file: %v", err)+ }+ t.Logf("updated golden file %s", golden)+ return+ }++ expected, err := os.ReadFile(golden)+ if err != nil {+ t.Fatalf("golden file %s missing — run with -update flag to generate: %v", golden, err)+ }++ if string(got) != string(expected) {+ t.Errorf("output differs from golden file %s\n\nGot:\n%s\n\nExpected:\n%s", golden, string(got), string(expected))+ }+ })+ }+}++// --- Unit tests ---++func TestHasHTTPAttributes(t *testing.T) {+ tests := []struct {+ name string+ attrs []*commonpb.KeyValue+ want bool+ }{+ {"with http.route", makeAttrs("http.route", "/api/users"), true},+ {"with http.request.method", makeAttrs("http.request.method", "GET"), true},+ {"with http.method", makeAttrs("http.method", "POST"), true},+ {"with url.path", makeAttrs("url.path", "/api"), true},+ {"without http attrs", makeAttrs("db.operation.name", "findOne"), false},+ {"empty", nil, false},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := hasHTTPAttributes(tt.attrs); got != tt.want {+ t.Errorf("hasHTTPAttributes() = %v, want %v", got, tt.want)+ }+ })+ }+}++func TestHasGenAiAttributes(t *testing.T) {+ tests := []struct {+ name string+ attrs []*commonpb.KeyValue+ want bool+ }{+ {"with gen_ai.request.model", makeAttrs("gen_ai.request.model", "gpt-4"), true},+ {"with gen_ai.usage.input_tokens", makeAttrs("gen_ai.usage.input_tokens", "50"), true},+ {"without gen_ai", makeAttrs("http.route", "/api"), false},+ {"empty", nil, false},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := hasGenAiAttributes(tt.attrs); got != tt.want {+ t.Errorf("hasGenAiAttributes() = %v, want %v", got, tt.want)+ }+ })+ }+}++func TestGetHTTPEndpoint(t *testing.T) {+ tests := []struct {+ name string+ attrs []*commonpb.KeyValue+ fallback string+ want string+ }{+ {"method+route", append(makeAttrs("http.request.method", "GET"), makeAttrs("http.route", "/api/users")...), "fallback", "GET /api/users"},+ {"old method+path", append(makeAttrs("http.method", "POST"), makeAttrs("url.path", "/submit")...), "fallback", "POST /submit"},+ {"method only", makeAttrs("http.request.method", "DELETE"), "my-op", "DELETE my-op"},+ {"no attrs", nil, "fallback", "fallback"},+ {"route without leading slash ignored", append(makeAttrs("http.request.method", "GET"), makeAttrs("http.route", "no-slash")...), "fallback", "GET fallback"},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := getHTTPEndpoint(tt.attrs, tt.fallback); got != tt.want {+ t.Errorf("getHTTPEndpoint() = %q, want %q", got, tt.want)+ }+ })+ }+}++func TestFilterNonStandardAiAttrs(t *testing.T) {+ input := map[string]string{+ "gen_ai.request.model": "gpt-4",+ "gen_ai.prompt": "...",+ "gen_ai.usage.input_tokens": "50",+ "trace.name": "My Agent",+ "user.id": "u123",+ "custom.tag": "production",+ "gen_ai.request.temperature": "0.7",+ }+ result := filterNonStandardAiAttrs(input)++ if _, ok := result["gen_ai.request.model"]; ok {+ t.Error("should not include gen_ai.request.model")+ }+ if _, ok := result["gen_ai.prompt"]; ok {+ t.Error("should not include gen_ai.prompt")+ }+ if _, ok := result["trace.name"]; ok {+ t.Error("should not include trace.name")+ }+ if v, ok := result["custom.tag"]; !ok || v != "production" {+ t.Error("should include custom.tag")+ }+ if v, ok := result["gen_ai.request.temperature"]; !ok || v != "0.7" {+ t.Error("should include gen_ai.request.temperature")+ }+}++func TestExtractConversation(t *testing.T) {+ t.Run("with prompt and completion", func(t *testing.T) {+ attrs := append(+ makeAttrs("gen_ai.prompt", `{"messages":[{"role":"user","content":"hello"}]}`),+ makeAttrs("gen_ai.completion", `{"choices":[{"message":{"content":"hi"}}]}`)...,+ )+ conv := extractConversation(attrs, testProjectId, uuid.New())+ if conv == nil {+ t.Fatal("expected conversation, got nil")+ }+ if len(conv.Content) == 0 {+ t.Error("expected non-empty content")+ }+ })++ t.Run("without prompt or completion", func(t *testing.T) {+ attrs := makeAttrs("gen_ai.request.model", "gpt-4")+ conv := extractConversation(attrs, testProjectId, uuid.New())+ if conv != nil {+ t.Error("expected nil conversation")+ }+ })+}++func TestTraceIdResolution_CrossScope(t *testing.T) {+ // Simulate: child in scope 1 is processed before parent in scope 2+ rootSpanId := []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}+ childSpanId := []byte{0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18}+ grandchildSpanId := []byte{0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28}+ traceIdBytes := []byte{0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99}++ now := uint64(1700000000000000000)++ req := &coltracepb.ExportTraceServiceRequest{+ ResourceSpans: []*tracepb.ResourceSpans{+ {+ Resource: &resourcepb.Resource{+ Attributes: []*commonpb.KeyValue{+ strKV("service.name", "test"),+ },+ },+ ScopeSpans: []*tracepb.ScopeSpans{+ {+ // Scope 1: grandchild arrives FIRST+ Spans: []*tracepb.Span{+ {+ TraceId: traceIdBytes,+ SpanId: grandchildSpanId,+ ParentSpanId: childSpanId,+ Name: "tcp.connect",+ Kind: tracepb.Span_SPAN_KIND_INTERNAL,+ StartTimeUnixNano: now,+ EndTimeUnixNano: now + 1000000,+ },+ },+ },+ {+ // Scope 2: child and root arrive AFTER+ Spans: []*tracepb.Span{+ {+ TraceId: traceIdBytes,+ SpanId: childSpanId,+ ParentSpanId: rootSpanId,+ Name: "db query",+ Kind: tracepb.Span_SPAN_KIND_INTERNAL,+ StartTimeUnixNano: now,+ EndTimeUnixNano: now + 2000000,+ },+ {+ TraceId: traceIdBytes,+ SpanId: rootSpanId,+ Name: "GET /api/test",+ Kind: tracepb.Span_SPAN_KIND_SERVER,+ StartTimeUnixNano: now,+ EndTimeUnixNano: now + 5000000,+ Attributes: []*commonpb.KeyValue{+ strKV("http.request.method", "GET"),+ strKV("http.route", "/api/test"),+ },+ },+ },+ },+ },+ },+ },+ }++ endpoints, _, spans, _, _, _ := convertTraces(testProjectId, req)++ if len(endpoints) != 1 {+ t.Fatalf("expected 1 endpoint, got %d", len(endpoints))+ }++ rootTraceId := endpoints[0].Id+ for _, s := range spans {+ if s.TraceId != rootTraceId {+ t.Errorf("span %q has traceId %s, want %s (root endpoint)", s.Name, s.TraceId, rootTraceId)+ }+ }+}++// --- Helpers ---++func makeAttrs(key, val string) []*commonpb.KeyValue {+ return []*commonpb.KeyValue{strKV(key, val)}+}++func strKV(key, val string) *commonpb.KeyValue {+ return &commonpb.KeyValue{+ Key: key,+ Value: &commonpb.AnyValue{Value: &commonpb.AnyValue_StringValue{StringValue: val}},+ }+}diff --git a/docs/pages/learn/contributing.mdx b/docs/pages/learn/contributing.mdxindex 5811c8ab..1673c47a 100644--- a/docs/pages/learn/contributing.mdx+++ b/docs/pages/learn/contributing.mdx@@ -122,6 +122,24 @@ This builds a Docker image with both databases, applies all migrations, and runsWithout `TEST_CLICKHOUSE_SERVER` set, the pgch tests skip cleanly.</Callout>+### OTEL trace converter tests++These test the OTLP trace parsing logic using snapshot/golden-file testing. No database required — `convertTraces()` is a pure function.++```bash+cd backend+go test -v -count=1 ./app/controllers/otelcontrollers/+```++Real OTEL JSON payloads are stored in `testdata/` as fixtures. The tests parse them through the converter and compare the output against `.golden.json` files. If you intentionally change the converter, regenerate the golden files:++```bash+cd backend+go test -v -count=1 -args -update ./app/controllers/otelcontrollers/+```++Then review the diff in the `.golden.json` files to confirm the changes are expected before committing.+### Frontend checks```bash@@ -213,13 +231,19 @@ go build -tags pgch ./... # ClickHouse + PostgreSQLgo test -v -count=1 ./app/repositories/```-3. **Run the pgch Docker tests** if you changed any repository code:+3. **Run the OTEL converter tests** if you changed any OTEL parsing code:++```bash+go test -v -count=1 ./app/controllers/otelcontrollers/+```++4. **Run the pgch Docker tests** if you changed any repository code:```bash./scripts/test-backend-pgch.sh```-4. **Check the frontend** if you changed any frontend code:+5. **Check the frontend** if you changed any frontend code:```bashcd frontenddiff --git a/frontend/src/lib/components/spans/span-row.svelte b/frontend/src/lib/components/spans/span-row.svelteindex 21e75a02..12c3721d 100644--- a/frontend/src/lib/components/spans/span-row.svelte+++ b/frontend/src/lib/components/spans/span-row.svelte@@ -148,7 +148,7 @@spanColor.bg,isHovered && `ring-2 ${spanColor.ring}`)}- style="left: {leftPercent}%; width: {Math.max(widthPercent, 1)}%"+ style="left: {leftPercent}%; width: {Math.max(widthPercent, 0.3)}%; min-width: 2px"onmouseenter={handleMouseEnter}onmouseleave={handleMouseLeave}role="presentation"diff --git a/frontend/src/lib/components/spans/span-waterfall.svelte b/frontend/src/lib/components/spans/span-waterfall.svelteindex 4f283690..f421cca6 100644--- a/frontend/src/lib/components/spans/span-waterfall.svelte+++ b/frontend/src/lib/components/spans/span-waterfall.svelte@@ -10,7 +10,11 @@traceStartTime: string;};- let { spans, traceDuration, traceStartTime }: Props = $props();+ let { spans: rawSpans, traceDuration, traceStartTime }: Props = $props();++ const spans = $derived(+ [...rawSpans].sort((a, b) => new Date(a.startTime).getTime() - new Date(b.startTime).getTime())+ );const traceStart = $derived(spans.length === 0