Skip to content

The only tool you need to know what is happening and how to fix it.

fix: short term patch before the long term one for traceid/spanid poor database field storage

narrow-adventurebbfe1aee08e2
diff --git a/backend/app/controllers/otelcontrollers/log_correlation_test.go b/backend/app/controllers/otelcontrollers/log_correlation_test.go
new file mode 100644
index 00000000..1562ff4d
--- /dev/null
+++ b/backend/app/controllers/otelcontrollers/log_correlation_test.go
@@ -0,0 +1,85 @@
+package otelcontrollers
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/hex"
+ "testing"
+
+ "github.com/google/uuid"
+ coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+ commonpb "go.opentelemetry.io/proto/otlp/common/v1"
+ logspb "go.opentelemetry.io/proto/otlp/logs/v1"
+ tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
+)
+
+func TestPromotedEntityLogCorrelation(t *testing.T) {
+ const traceHex = "8462dd06157032df3c6c90ac5ff74ed7"
+ traceBytes, _ := hex.DecodeString(traceHex)
+ jsonTraceBytes, _ := base64.StdEncoding.DecodeString(traceHex)
+ spanBytes, _ := hex.DecodeString("1985a7abed0024db")
+ groupID := uuid.MustParse("a5100000-0000-4000-8000-000000000001")
+
+ for _, entity := range []struct {
+ name string
+ kind tracepb.Span_SpanKind
+ attrs []*commonpb.KeyValue
+ }{
+ {"task", tracepb.Span_SPAN_KIND_CONSUMER, nil},
+ {"endpoint", tracepb.Span_SPAN_KIND_SERVER, []*commonpb.KeyValue{strKV("http.request.method", "GET")}},
+ {"ai", tracepb.Span_SPAN_KIND_CLIENT, []*commonpb.KeyValue{strKV("gen_ai.system", "openai")}},
+ } {
+ for _, encoding := range []struct {
+ name string
+ id []byte
+ }{
+ {"protobuf", traceBytes},
+ {"json", jsonTraceBytes},
+ } {
+ for _, override := range []bool{false, true} {
+ t.Run(entity.name+"/"+encoding.name+"/override="+map[bool]string{false: "false", true: "true"}[override], func(t *testing.T) {
+ attrs := append([]*commonpb.KeyValue{}, entity.attrs...)
+ if override {
+ attrs = append(attrs, strKV("traceway.distributed_trace_id", groupID.String()))
+ }
+ req := &coltracepb.ExportTraceServiceRequest{ResourceSpans: []*tracepb.ResourceSpans{{
+ ScopeSpans: []*tracepb.ScopeSpans{{Spans: []*tracepb.Span{{
+ TraceId: encoding.id, SpanId: spanBytes, ParentSpanId: []byte{1, 2, 3, 4, 5, 6, 7, 8},
+ Name: "worker", Kind: entity.kind, Attributes: attrs,
+ StartTimeUnixNano: 1_700_000_000_000_000_000, EndTimeUnixNano: 1_700_000_000_001_000_000,
+ }}}},
+ }}}
+ endpoints, tasks, _, _, aiTraces, _ := convertTraces(context.Background(), nil, testProjectId, req)
+ if len(endpoints)+len(tasks)+len(aiTraces) != 1 {
+ t.Fatal("expected one promoted entity")
+ }
+ var id uuid.UUID
+ var distributedID *uuid.UUID
+ var attributes map[string]string
+ switch entity.name {
+ case "task":
+ id, distributedID, attributes = tasks[0].Id, tasks[0].DistributedTraceId, tasks[0].Attributes
+ case "endpoint":
+ id, distributedID, attributes = endpoints[0].Id, endpoints[0].DistributedTraceId, endpoints[0].Attributes
+ case "ai":
+ id, distributedID, attributes = aiTraces[0].Id, aiTraces[0].DistributedTraceId, aiTraces[0].Attributes
+ }
+ log := toLogRecord(testProjectId, &logspb.LogRecord{TraceId: encoding.id, SpanId: spanBytes}, "worker", "", nil, "", "", "", nil)
+ if attributes["traceway.otel.trace_id"] != traceHex || attributes["traceway.otel.trace_id"] != log.TraceId {
+ t.Fatalf("entity trace ID %q does not correlate with log trace ID %q", attributes["traceway.otel.trace_id"], log.TraceId)
+ }
+ if id != otelSpanIDToUUID(spanBytes) {
+ t.Fatalf("occurrence ID changed: %s", id)
+ }
+ wantGroup := otelTraceIDToUUID(traceBytes)
+ if override {
+ wantGroup = groupID
+ }
+ if distributedID == nil || *distributedID != wantGroup {
+ t.Fatalf("distributed grouping changed: %v", distributedID)
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/backend/app/controllers/otelcontrollers/trace_converter.go b/backend/app/controllers/otelcontrollers/trace_converter.go
index 3836bcb7..2d7c49a1 100644
--- a/backend/app/controllers/otelcontrollers/trace_converter.go
+++ b/backend/app/controllers/otelcontrollers/trace_converter.go
@@ -2,6 +2,7 @@ package otelcontrollers
import (
"context"
+ "encoding/hex"
"encoding/json"
"fmt"
"net/http"
@@ -217,6 +218,14 @@ func convertTraces(ctx context.Context, existingProject *models.Project, project
}
if promoted {
+ // Occurrence IDs can come from span_id, and distributed_trace_id can be
+ // overridden. Keep the wire trace ID for correlation with OTLP logs.
+ if traceId := otelTraceIDToUUID(span.TraceId); traceId != uuid.Nil {
+ if allAttrs == nil {
+ allAttrs = make(map[string]string)
+ }
+ allAttrs["traceway.otel.trace_id"] = hex.EncodeToString(traceId[:])
+ }
rootSpanId := otelSpanIDToUUID(span.SpanId)
switch prom.kind {
case entityEndpoint:
diff --git a/frontend/src/lib/utils/span-id.test.ts b/frontend/src/lib/utils/span-id.test.ts
new file mode 100644
index 00000000..996b4321
--- /dev/null
+++ b/frontend/src/lib/utils/span-id.test.ts
@@ -0,0 +1,30 @@
+import { describe, expect, it } from 'vitest';
+import { logTraceId } from './span-id';
+
+const traceId = '8462dd06-1570-32df-3c6c-90ac5ff74ed7';
+const traceHex = '8462dd06157032df3c6c90ac5ff74ed7';
+const occurrenceId = '00000000-0000-0000-1985-a7abed0024db';
+const groupId = 'a5100000-0000-4000-8000-000000000001';
+
+describe('logTraceId', () => {
+ it('correlates existing non-root occurrences without requiring a stored upstream root', () => {
+ expect(logTraceId({ id: occurrenceId, distributedTraceId: traceId })).toBe(traceHex);
+ });
+
+ it('keeps native trace IDs independent of distributed grouping', () => {
+ expect(logTraceId({ id: traceId, distributedTraceId: groupId })).toBe(traceHex);
+ expect(logTraceId({ id: traceId, attributes: null })).toBe(traceHex);
+ });
+
+ it('uses the original OTel ID even when distributed grouping is overridden', () => {
+ for (const id of [occurrenceId, traceId]) {
+ expect(
+ logTraceId({
+ id,
+ distributedTraceId: groupId,
+ attributes: { 'traceway.otel.trace_id': traceHex.toUpperCase() }
+ })
+ ).toBe(traceHex);
+ }
+ });
+});
diff --git a/frontend/src/lib/utils/span-id.ts b/frontend/src/lib/utils/span-id.ts
index 3922968f..34515527 100644
--- a/frontend/src/lib/utils/span-id.ts
+++ b/frontend/src/lib/utils/span-id.ts
@@ -7,3 +7,20 @@ export function traceIdUuidToHex(traceUuid: string | null | undefined): string {
if (!traceUuid) return '';
return traceUuid.replace(/-/g, '').toLowerCase();
}
+
+export function logTraceId(trace: {
+ id: string;
+ distributedTraceId?: string | null;
+ attributes?: Record<string, string> | null;
+}): string {
+ const otelTraceId = trace.attributes?.['traceway.otel.trace_id'];
+ if (otelTraceId) return traceIdUuidToHex(otelTraceId);
+
+ const occurrenceId = traceIdUuidToHex(trace.id);
+ // Older OTel non-root occurrences store a zero-padded span ID as their ID.
+ // Native traces keep their own trace ID, even when grouped into a distributed trace.
+ if (occurrenceId.startsWith('0000000000000000') && trace.distributedTraceId) {
+ return traceIdUuidToHex(trace.distributedTraceId);
+ }
+ return occurrenceId;
+}
diff --git a/frontend/src/routes/ai-traces/[traceName]/[traceId]/+page.svelte b/frontend/src/routes/ai-traces/[traceName]/[traceId]/+page.svelte
index 23f64c60..a2692937 100644
--- a/frontend/src/routes/ai-traces/[traceName]/[traceId]/+page.svelte
+++ b/frontend/src/routes/ai-traces/[traceName]/[traceId]/+page.svelte
@@ -13,7 +13,7 @@
import { createSmartBackHandler } from '$lib/utils/back-navigation';
import { resolve } from '$app/paths';
import TraceLogsPanel from '$lib/components/trace-logs/trace-logs-panel.svelte';
- import { traceIdUuidToHex } from '$lib/utils/span-id';
+ import { logTraceId } from '$lib/utils/span-id';
import { formatCost } from '$lib/utils/ai-format';
import { extractMessages, formatConversationContent } from '$lib/utils/ai-conversation';
import ConversationMessages from '$lib/components/ai/conversation-messages.svelte';
@@ -284,7 +284,7 @@
<TraceLogsPanel
projectId={projectsState.currentProjectId ?? ''}
- traceId={traceIdUuidToHex(trace.id)}
+ traceId={logTraceId(trace)}
distributedTraceId={trace.distributedTraceId ?? null}
spans={[]}
traceRecordedAt={trace.recordedAt}
diff --git a/frontend/src/routes/endpoints/[endpoint]/[endpointId]/+page.svelte b/frontend/src/routes/endpoints/[endpoint]/[endpointId]/+page.svelte
index 3c3c7ec0..6c8a6a44 100644
--- a/frontend/src/routes/endpoints/[endpoint]/[endpointId]/+page.svelte
+++ b/frontend/src/routes/endpoints/[endpoint]/[endpointId]/+page.svelte
@@ -23,7 +23,7 @@
import { resolve } from '$app/paths';
import DistributedTraceCard from '$lib/components/distributed-trace/distributed-trace-card.svelte';
import TraceLogsPanel from '$lib/components/trace-logs/trace-logs-panel.svelte';
- import { traceIdUuidToHex } from '$lib/utils/span-id';
+ import { logTraceId } from '$lib/utils/span-id';
let { data } = $props();
@@ -278,7 +278,7 @@
<TraceLogsPanel
projectId={projectsState.currentProjectId ?? ''}
- traceId={traceIdUuidToHex(response.endpoint.id)}
+ traceId={logTraceId(response.endpoint)}
distributedTraceId={response.endpoint.distributedTraceId ?? null}
spans={response.spans ?? []}
traceRecordedAt={response.endpoint.recordedAt}
diff --git a/frontend/src/routes/tasks/[task]/[taskId]/+page.svelte b/frontend/src/routes/tasks/[task]/[taskId]/+page.svelte
index f526b4a4..2b6f89b5 100644
--- a/frontend/src/routes/tasks/[task]/[taskId]/+page.svelte
+++ b/frontend/src/routes/tasks/[task]/[taskId]/+page.svelte
@@ -22,7 +22,7 @@
import { resolve } from '$app/paths';
import DistributedTraceCard from '$lib/components/distributed-trace/distributed-trace-card.svelte';
import TraceLogsPanel from '$lib/components/trace-logs/trace-logs-panel.svelte';
- import { traceIdUuidToHex } from '$lib/utils/span-id';
+ import { logTraceId } from '$lib/utils/span-id';
import type { Span } from '$lib/types/spans';
type TaskDetailResponse = {
@@ -278,7 +278,7 @@
<TraceLogsPanel
projectId={projectsState.currentProjectId ?? ''}
- traceId={traceIdUuidToHex(response.task.id)}
+ traceId={logTraceId(response.task)}
distributedTraceId={response.task.distributedTraceId ?? null}
spans={response.spans ?? []}
traceRecordedAt={response.task.recordedAt}