The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
fix: short term patch before the long term one for traceid/spanid poor database field storage
narrow-adventurebbfe1aee08e2
diff --git a/backend/app/controllers/otelcontrollers/log_correlation_test.go b/backend/app/controllers/otelcontrollers/log_correlation_test.gonew file mode 100644index 00000000..1562ff4d--- /dev/null+++ b/backend/app/controllers/otelcontrollers/log_correlation_test.go@@ -0,0 +1,85 @@+package otelcontrollers++import (+ "context"+ "encoding/base64"+ "encoding/hex"+ "testing"++ "github.com/google/uuid"+ coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"+ commonpb "go.opentelemetry.io/proto/otlp/common/v1"+ logspb "go.opentelemetry.io/proto/otlp/logs/v1"+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"+)++func TestPromotedEntityLogCorrelation(t *testing.T) {+ const traceHex = "8462dd06157032df3c6c90ac5ff74ed7"+ traceBytes, _ := hex.DecodeString(traceHex)+ jsonTraceBytes, _ := base64.StdEncoding.DecodeString(traceHex)+ spanBytes, _ := hex.DecodeString("1985a7abed0024db")+ groupID := uuid.MustParse("a5100000-0000-4000-8000-000000000001")++ for _, entity := range []struct {+ name string+ kind tracepb.Span_SpanKind+ attrs []*commonpb.KeyValue+ }{+ {"task", tracepb.Span_SPAN_KIND_CONSUMER, nil},+ {"endpoint", tracepb.Span_SPAN_KIND_SERVER, []*commonpb.KeyValue{strKV("http.request.method", "GET")}},+ {"ai", tracepb.Span_SPAN_KIND_CLIENT, []*commonpb.KeyValue{strKV("gen_ai.system", "openai")}},+ } {+ for _, encoding := range []struct {+ name string+ id []byte+ }{+ {"protobuf", traceBytes},+ {"json", jsonTraceBytes},+ } {+ for _, override := range []bool{false, true} {+ t.Run(entity.name+"/"+encoding.name+"/override="+map[bool]string{false: "false", true: "true"}[override], func(t *testing.T) {+ attrs := append([]*commonpb.KeyValue{}, entity.attrs...)+ if override {+ attrs = append(attrs, strKV("traceway.distributed_trace_id", groupID.String()))+ }+ req := &coltracepb.ExportTraceServiceRequest{ResourceSpans: []*tracepb.ResourceSpans{{+ ScopeSpans: []*tracepb.ScopeSpans{{Spans: []*tracepb.Span{{+ TraceId: encoding.id, SpanId: spanBytes, ParentSpanId: []byte{1, 2, 3, 4, 5, 6, 7, 8},+ Name: "worker", Kind: entity.kind, Attributes: attrs,+ StartTimeUnixNano: 1_700_000_000_000_000_000, EndTimeUnixNano: 1_700_000_000_001_000_000,+ }}}},+ }}}+ endpoints, tasks, _, _, aiTraces, _ := convertTraces(context.Background(), nil, testProjectId, req)+ if len(endpoints)+len(tasks)+len(aiTraces) != 1 {+ t.Fatal("expected one promoted entity")+ }+ var id uuid.UUID+ var distributedID *uuid.UUID+ var attributes map[string]string+ switch entity.name {+ case "task":+ id, distributedID, attributes = tasks[0].Id, tasks[0].DistributedTraceId, tasks[0].Attributes+ case "endpoint":+ id, distributedID, attributes = endpoints[0].Id, endpoints[0].DistributedTraceId, endpoints[0].Attributes+ case "ai":+ id, distributedID, attributes = aiTraces[0].Id, aiTraces[0].DistributedTraceId, aiTraces[0].Attributes+ }+ log := toLogRecord(testProjectId, &logspb.LogRecord{TraceId: encoding.id, SpanId: spanBytes}, "worker", "", nil, "", "", "", nil)+ if attributes["traceway.otel.trace_id"] != traceHex || attributes["traceway.otel.trace_id"] != log.TraceId {+ t.Fatalf("entity trace ID %q does not correlate with log trace ID %q", attributes["traceway.otel.trace_id"], log.TraceId)+ }+ if id != otelSpanIDToUUID(spanBytes) {+ t.Fatalf("occurrence ID changed: %s", id)+ }+ wantGroup := otelTraceIDToUUID(traceBytes)+ if override {+ wantGroup = groupID+ }+ if distributedID == nil || *distributedID != wantGroup {+ t.Fatalf("distributed grouping changed: %v", distributedID)+ }+ })+ }+ }+ }+}diff --git a/backend/app/controllers/otelcontrollers/trace_converter.go b/backend/app/controllers/otelcontrollers/trace_converter.goindex 3836bcb7..2d7c49a1 100644--- a/backend/app/controllers/otelcontrollers/trace_converter.go+++ b/backend/app/controllers/otelcontrollers/trace_converter.go@@ -2,6 +2,7 @@ package otelcontrollersimport ("context"+ "encoding/hex""encoding/json""fmt""net/http"@@ -217,6 +218,14 @@ func convertTraces(ctx context.Context, existingProject *models.Project, project}if promoted {+ // Occurrence IDs can come from span_id, and distributed_trace_id can be+ // overridden. Keep the wire trace ID for correlation with OTLP logs.+ if traceId := otelTraceIDToUUID(span.TraceId); traceId != uuid.Nil {+ if allAttrs == nil {+ allAttrs = make(map[string]string)+ }+ allAttrs["traceway.otel.trace_id"] = hex.EncodeToString(traceId[:])+ }rootSpanId := otelSpanIDToUUID(span.SpanId)switch prom.kind {case entityEndpoint:diff --git a/frontend/src/lib/utils/span-id.test.ts b/frontend/src/lib/utils/span-id.test.tsnew file mode 100644index 00000000..996b4321--- /dev/null+++ b/frontend/src/lib/utils/span-id.test.ts@@ -0,0 +1,30 @@+import { describe, expect, it } from 'vitest';+import { logTraceId } from './span-id';++const traceId = '8462dd06-1570-32df-3c6c-90ac5ff74ed7';+const traceHex = '8462dd06157032df3c6c90ac5ff74ed7';+const occurrenceId = '00000000-0000-0000-1985-a7abed0024db';+const groupId = 'a5100000-0000-4000-8000-000000000001';++describe('logTraceId', () => {+ it('correlates existing non-root occurrences without requiring a stored upstream root', () => {+ expect(logTraceId({ id: occurrenceId, distributedTraceId: traceId })).toBe(traceHex);+ });++ it('keeps native trace IDs independent of distributed grouping', () => {+ expect(logTraceId({ id: traceId, distributedTraceId: groupId })).toBe(traceHex);+ expect(logTraceId({ id: traceId, attributes: null })).toBe(traceHex);+ });++ it('uses the original OTel ID even when distributed grouping is overridden', () => {+ for (const id of [occurrenceId, traceId]) {+ expect(+ logTraceId({+ id,+ distributedTraceId: groupId,+ attributes: { 'traceway.otel.trace_id': traceHex.toUpperCase() }+ })+ ).toBe(traceHex);+ }+ });+});diff --git a/frontend/src/lib/utils/span-id.ts b/frontend/src/lib/utils/span-id.tsindex 3922968f..34515527 100644--- a/frontend/src/lib/utils/span-id.ts+++ b/frontend/src/lib/utils/span-id.ts@@ -7,3 +7,20 @@ export function traceIdUuidToHex(traceUuid: string | null | undefined): string {if (!traceUuid) return '';return traceUuid.replace(/-/g, '').toLowerCase();}++export function logTraceId(trace: {+ id: string;+ distributedTraceId?: string | null;+ attributes?: Record<string, string> | null;+}): string {+ const otelTraceId = trace.attributes?.['traceway.otel.trace_id'];+ if (otelTraceId) return traceIdUuidToHex(otelTraceId);++ const occurrenceId = traceIdUuidToHex(trace.id);+ // Older OTel non-root occurrences store a zero-padded span ID as their ID.+ // Native traces keep their own trace ID, even when grouped into a distributed trace.+ if (occurrenceId.startsWith('0000000000000000') && trace.distributedTraceId) {+ return traceIdUuidToHex(trace.distributedTraceId);+ }+ return occurrenceId;+}diff --git a/frontend/src/routes/ai-traces/[traceName]/[traceId]/+page.svelte b/frontend/src/routes/ai-traces/[traceName]/[traceId]/+page.svelteindex 23f64c60..a2692937 100644--- a/frontend/src/routes/ai-traces/[traceName]/[traceId]/+page.svelte+++ b/frontend/src/routes/ai-traces/[traceName]/[traceId]/+page.svelte@@ -13,7 +13,7 @@import { createSmartBackHandler } from '$lib/utils/back-navigation';import { resolve } from '$app/paths';import TraceLogsPanel from '$lib/components/trace-logs/trace-logs-panel.svelte';- import { traceIdUuidToHex } from '$lib/utils/span-id';+ import { logTraceId } from '$lib/utils/span-id';import { formatCost } from '$lib/utils/ai-format';import { extractMessages, formatConversationContent } from '$lib/utils/ai-conversation';import ConversationMessages from '$lib/components/ai/conversation-messages.svelte';@@ -284,7 +284,7 @@<TraceLogsPanelprojectId={projectsState.currentProjectId ?? ''}- traceId={traceIdUuidToHex(trace.id)}+ traceId={logTraceId(trace)}distributedTraceId={trace.distributedTraceId ?? null}spans={[]}traceRecordedAt={trace.recordedAt}diff --git a/frontend/src/routes/endpoints/[endpoint]/[endpointId]/+page.svelte b/frontend/src/routes/endpoints/[endpoint]/[endpointId]/+page.svelteindex 3c3c7ec0..6c8a6a44 100644--- a/frontend/src/routes/endpoints/[endpoint]/[endpointId]/+page.svelte+++ b/frontend/src/routes/endpoints/[endpoint]/[endpointId]/+page.svelte@@ -23,7 +23,7 @@import { resolve } from '$app/paths';import DistributedTraceCard from '$lib/components/distributed-trace/distributed-trace-card.svelte';import TraceLogsPanel from '$lib/components/trace-logs/trace-logs-panel.svelte';- import { traceIdUuidToHex } from '$lib/utils/span-id';+ import { logTraceId } from '$lib/utils/span-id';let { data } = $props();@@ -278,7 +278,7 @@<TraceLogsPanelprojectId={projectsState.currentProjectId ?? ''}- traceId={traceIdUuidToHex(response.endpoint.id)}+ traceId={logTraceId(response.endpoint)}distributedTraceId={response.endpoint.distributedTraceId ?? null}spans={response.spans ?? []}traceRecordedAt={response.endpoint.recordedAt}diff --git a/frontend/src/routes/tasks/[task]/[taskId]/+page.svelte b/frontend/src/routes/tasks/[task]/[taskId]/+page.svelteindex f526b4a4..2b6f89b5 100644--- a/frontend/src/routes/tasks/[task]/[taskId]/+page.svelte+++ b/frontend/src/routes/tasks/[task]/[taskId]/+page.svelte@@ -22,7 +22,7 @@import { resolve } from '$app/paths';import DistributedTraceCard from '$lib/components/distributed-trace/distributed-trace-card.svelte';import TraceLogsPanel from '$lib/components/trace-logs/trace-logs-panel.svelte';- import { traceIdUuidToHex } from '$lib/utils/span-id';+ import { logTraceId } from '$lib/utils/span-id';import type { Span } from '$lib/types/spans';type TaskDetailResponse = {@@ -278,7 +278,7 @@<TraceLogsPanelprojectId={projectsState.currentProjectId ?? ''}- traceId={traceIdUuidToHex(response.task.id)}+ traceId={logTraceId(response.task)}distributedTraceId={response.task.distributedTraceId ?? null}spans={response.spans ?? []}traceRecordedAt={response.task.recordedAt}