The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
fix: migration to v2 fix
narrow-adventuree64abaff1818
diff --git a/backend/app/repositories/telemetry/move_over.go b/backend/app/repositories/telemetry/move_over.goindex 620e533b..67c67368 100644--- a/backend/app/repositories/telemetry/move_over.go+++ b/backend/app/repositories/telemetry/move_over.go@@ -355,7 +355,7 @@ func (m *mover) insertSpans(ctx context.Context, spans []models.Span, from, to tfunc (m *mover) writeEndpoints(ctx context.Context, rows []models.Endpoint, from, to time.Time, resumed bool) error {ids := make([]uuid.UUID, len(rows))for i, row := range rows {- ids[i] = row.Id+ ids[i] = legacyId(row.Id, row.ProjectId, row.RecordedAt, row.Duration, row.Endpoint)}moved, err := m.movedIds(ctx, "endpoints", ids, from, to, resumed)if err != nil {@@ -379,7 +379,7 @@ func (m *mover) writeEndpoints(ctx context.Context, rows []models.Endpoint, fromfunc (m *mover) writeTasks(ctx context.Context, rows []models.Task, from, to time.Time, resumed bool) error {ids := make([]uuid.UUID, len(rows))for i, row := range rows {- ids[i] = row.Id+ ids[i] = legacyId(row.Id, row.ProjectId, row.RecordedAt, row.Duration, row.TaskName)// Legacy OTel tasks stored their end time, with no upper bound on duration.if start := row.RecordedAt.Add(-row.Duration); row.SpanId != "" && start.Before(from) {from = start@@ -407,7 +407,7 @@ func (m *mover) writeTasks(ctx context.Context, rows []models.Task, from, to timfunc (m *mover) writeAiTraces(ctx context.Context, rows []models.AiTrace, from, to time.Time, resumed bool) error {ids := make([]uuid.UUID, len(rows))for i, row := range rows {- ids[i] = row.Id+ ids[i] = legacyId(row.Id, row.ProjectId, row.RecordedAt, row.Duration, row.TraceName)}moved, err := m.movedIds(ctx, "ai_traces", ids, from, to, resumed)if err != nil {@@ -545,16 +545,28 @@ const (func hexId(id uuid.UUID) string { return shared.NormalizeTraceId(id.String()) }+// legacyId stands in for the nil UUID, which the old ingest stored whenever a client sent it as a run or span id. The+// stand-in is derived from the row, so it differs between rows and is the same on every read of one.+func legacyId(id, projectId uuid.UUID, at time.Time, duration time.Duration, name string) uuid.UUID {+ if id != uuid.Nil {+ return id+ }+ return uuid.NewSHA1(projectId, fmt.Appendf(nil, "%d|%d|%s", at.UnixNano(), duration, name))+}+func validTraceId(id string) bool {return len(id) == 32 && id != zeroPadding+zeroPadding && strings.Trim(id, "0123456789abcdef") == ""}// spanHex undoes the old storage of an OTel span id, which was kept as a UUID with eight zero bytes in front. A native-// id is a real UUID and stays 32 characters.+// id is a real UUID and stays 32 characters. An all-zero id is no id.func spanHex(id string) string {id = shared.NormalizeTraceId(id)if len(id) == 32 && strings.HasPrefix(id, zeroPadding) {- return id[16:]+ id = id[16:]+ }+ if strings.Trim(id, "0") == "" {+ return ""}return id}@@ -580,6 +592,7 @@ func withoutIdentity(attributes map[string]string) map[string]string {}func mapEndpoint(row models.Endpoint) (models.Endpoint, models.Span) {+ row.Id = legacyId(row.Id, row.ProjectId, row.RecordedAt, row.Duration, row.Endpoint)row.TraceId = traceIdFromLegacy(row.Attributes, row.TraceId, row.Id)if row.SpanId = spanHex(row.SpanId); row.SpanId == "" {row.SpanId = hexId(row.Id)@@ -594,6 +607,7 @@ func mapEndpoint(row models.Endpoint) (models.Endpoint, models.Span) {}func mapTask(row models.Task) (models.Task, models.Span) {+ row.Id = legacyId(row.Id, row.ProjectId, row.RecordedAt, row.Duration, row.TaskName)row.TraceId = traceIdFromLegacy(row.Attributes, row.TraceId, row.Id)kind := int32(spanKindConsumer)if row.SpanId = spanHex(row.SpanId); row.SpanId == "" {@@ -608,6 +622,7 @@ func mapTask(row models.Task) (models.Task, models.Span) {}func mapAiTrace(row models.AiTrace) (models.AiTrace, models.Span) {+ row.Id = legacyId(row.Id, row.ProjectId, row.RecordedAt, row.Duration, row.TraceName)row.TraceId = traceIdFromLegacy(row.Attributes, row.TraceId, row.Id)// The old table kept no span id. A call below the root was stored under its span id. The root was stored under the// trace id, which serves as its span id here: it only has to be the same on the row and on the span.@@ -627,7 +642,7 @@ func mapException(legacy shared.LegacyException, owners ownerIndex) models.ExcepownerId, err := uuid.Parse(row.TraceId)row.Attributes, row.SpanId = withoutIdentity(row.Attributes), ""switch found, known := owners.nearest(row.ProjectId, ownerId, row.RecordedAt); {- case err != nil:+ case err != nil, ownerId == uuid.Nil && !known:row.TraceId, row.TraceType = distributed, ""case known:row.TraceId, row.SpanId = found.traceId, found.spanId@@ -638,15 +653,16 @@ func mapException(legacy shared.LegacyException, owners ownerIndex) models.Excep}func mapSpan(row shared.LegacySpan, owners ownerIndex) models.Span {- span := models.Span{ProjectId: row.ProjectId, TraceId: hexId(row.OwnerId), SpanId: spanHex(row.Id.String()), ParentSpanId: spanHex(row.ParentSpanId),- Name: row.Name, StartTime: row.StartTime, RecordedAt: row.RecordedAt, Duration: row.Duration, Attributes: withoutIdentity(row.Attributes)}+ run := hexId(legacyId(row.OwnerId, row.ProjectId, row.RecordedAt, 0, ""))+ span := models.Span{ProjectId: row.ProjectId, TraceId: run, SpanId: spanHex(legacyId(row.Id, row.ProjectId, row.StartTime, row.Duration, row.Name).String()),+ ParentSpanId: spanHex(row.ParentSpanId), Name: row.Name, StartTime: row.StartTime, RecordedAt: row.RecordedAt, Duration: row.Duration, Attributes: withoutIdentity(row.Attributes)}found, known := owners.nearest(row.ProjectId, row.OwnerId, row.RecordedAt)if known {span.TraceId, span.ServiceName = found.traceId, found.serverName}if span.ParentSpanId == "" {// Only a native span was stored without a parent. It hangs under its run, whose span id is the run's own id.- if span.ParentSpanId = hexId(row.OwnerId); known {+ if span.ParentSpanId = run; known {span.ParentSpanId = found.spanId}}diff --git a/backend/app/repositories/telemetry/move_over_test.go b/backend/app/repositories/telemetry/move_over_test.goindex dd3dddcf..a0136b56 100644--- a/backend/app/repositories/telemetry/move_over_test.go+++ b/backend/app/repositories/telemetry/move_over_test.go@@ -144,6 +144,9 @@ func TestMoveOverIds(t *testing.T) {if got := spanHex(""); got != "" {t.Errorf("no id stays no id: %q", got)}+ if got := spanHex(uuid.Nil.String()); got != "" {+ t.Errorf("a zero id is no id: %q", got)+ }recorded := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC)task, span := mapTask(models.Task{Id: paddedSpan(5), TaskName: "settle", TraceId: trace.String(), SpanId: paddedSpan(5).String(), RecordedAt: recorded, Duration: time.Minute,@@ -226,6 +229,60 @@ func TestMoveOverZeroDistributedIDKeepsNativeGraph(t *testing.T) {}}+func TestMoveOverNilIdsGetStandIns(t *testing.T) {+ setupTestDB(t)+ setupMoveOverProgress(t)+ legacyReset(t)+ t.Cleanup(func() { legacyReset(t) })+ ctx := context.Background()+ at := time.Now().UTC().Truncate(moveOverDay).Add(10 * time.Hour)+ project, exception := uuid.New(), uuid.New()+ for _, recorded := range []time.Time{at, at.Add(time.Second)} {+ legacyExec(t, `INSERT INTO endpoints (id, project_id, endpoint, duration, recorded_at, status_code, body_size, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_stream, is_root)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, uuid.Nil, project, "GET /nil", int64(time.Second), recorded, 200, 0, "", "{}", "1.0", "api", (*uuid.UUID)(nil), (*uuid.UUID)(nil), false, true)+ }+ legacyExec(t, `INSERT INTO spans (id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id, attributes)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, uuid.Nil, uuid.Nil, project, "child", at, int64(time.Millisecond), at, &uuid.Nil, "{}")+ legacyExec(t, `INSERT INTO exception_stack_traces (id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_id)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, exception, project, &uuid.Nil, "endpoint", "nil-run", "Error: nil run", at, "{}", "1.0", "api", false, (*uuid.UUID)(nil), (*uuid.UUID)(nil))+ if err := RunMoveOver(ctx, MoveOverOptions{Log: func(string, ...any) {}}); err != nil {+ t.Fatal(err)+ }+ first, second := legacyId(uuid.Nil, project, at, time.Second, "GET /nil"), legacyId(uuid.Nil, project, at.Add(time.Second), time.Second, "GET /nil")+ if first == uuid.Nil || first == second {+ t.Fatalf("each nil run needs its own stand-in: %s %s", first, second)+ }+ endpoint, err := EndpointRepository.FindById(ctx, project, first, &at)+ if err != nil || endpoint == nil || endpoint.TraceId != hexId(first) || endpoint.SpanId != hexId(first) {+ t.Fatalf("migrated endpoint: %+v %v", endpoint, err)+ }+ if got := moveOverRowCount(t, "endpoints_v2"); got != 2 {+ t.Fatalf("endpoints_v2 rows = %d, want 2", got)+ }+ graph, err := SpanRepository.FindTrace(ctx, []uuid.UUID{project}, hexId(first), at)+ if err != nil || len(graph.Spans) != 2 {+ t.Fatalf("migrated root and child: %+v %v", graph, err)+ }+ for _, span := range graph.Spans {+ if span.SpanId != hexId(first) && span.ParentSpanId != hexId(first) {+ t.Fatalf("a child with a nil parent hangs under its run: %+v", span)+ }+ }+ exc, err := ExceptionStackTraceRepository.FindById(ctx, project, exception, &at)+ if err != nil || exc == nil || exc.TraceId != hexId(first) || exc.SpanId != hexId(first) {+ t.Fatalf("migrated exception: %+v %v", exc, err)+ }++ lost := mapSpan(shared.LegacySpan{ProjectId: project, Id: uuid.Nil, OwnerId: uuid.Nil, ParentSpanId: uuid.Nil.String(), RecordedAt: at}, ownerIndex{})+ if !validTraceId(lost.TraceId) || lost.SpanId == "" || lost.ParentSpanId != lost.TraceId {+ t.Errorf("a span whose nil run is gone still gets valid ids: %+v", lost)+ }+ unowned := mapException(shared.LegacyException{ExceptionStackTrace: models.ExceptionStackTrace{ProjectId: project, TraceId: uuid.Nil.String(), TraceType: "endpoint"}}, ownerIndex{})+ if unowned.TraceId != "" || unowned.TraceType != "" {+ t.Errorf("an exception whose nil run is gone claims no trace: %+v", unowned)+ }+}+// seedLegacyDay writes one request the way the released ingest stored it: a root endpoint filed under the trace id, a// worker task below it filed under its span id, their spans filed under their owner, and an exception on the task.func seedLegacyDay(t *testing.T, project uuid.UUID, at time.Time, trace uuid.UUID, endpointName string) {