The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
fix: progress
narrow-adventure6b3228da619e
diff --git a/backend/app/controllers/distributed_trace.controller.go b/backend/app/controllers/distributed_trace.controller.goindex eff605db..c5aead49 100644--- a/backend/app/controllers/distributed_trace.controller.go+++ b/backend/app/controllers/distributed_trace.controller.go@@ -10,6 +10,7 @@ import ("github.com/tracewayapp/traceway/backend/app/middleware""github.com/tracewayapp/traceway/backend/app/models""github.com/tracewayapp/traceway/backend/app/repositories/telemetry"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared""github.com/tracewayapp/traceway/backend/app/repositories/transactional""github.com/gin-gonic/gin"@@ -75,7 +76,7 @@ func (d distributedTraceController) GetDistributedTrace(c *gin.Context) {projectNameMap[p.Id] = p.Name}- ctx := context.Background()+ ctx := c.Request.Context()endpoints, err := telemetry.EndpointRepository.FindByDistributedTraceId(ctx, distributedTraceId, projectIds, request.RecordedAt)if err != nil {@@ -101,11 +102,11 @@ func (d distributedTraceController) GetDistributedTrace(c *gin.Context) {return}- exceptionByTraceId := make(map[uuid.UUID]*EndpointExceptionInfo)+ exceptionByTraceId := make(map[distributedTraceOwner]*EndpointExceptionInfo)for _, exc := range exceptions {if exc.TraceId != nil {- if _, exists := exceptionByTraceId[*exc.TraceId]; !exists {- exceptionByTraceId[*exc.TraceId] = &EndpointExceptionInfo{+ if _, exists := exceptionByTraceId[distributedTraceOwner{exc.ProjectId, *exc.TraceId}]; !exists {+ exceptionByTraceId[distributedTraceOwner{exc.ProjectId, *exc.TraceId}] = &EndpointExceptionInfo{ExceptionHash: exc.ExceptionHash,StackTrace: exc.StackTrace,RecordedAt: exc.RecordedAt.Format("2006-01-02T15:04:05Z07:00"),@@ -114,18 +115,19 @@ func (d distributedTraceController) GetDistributedTrace(c *gin.Context) {}}- matchedIds := make(map[uuid.UUID]bool)+ matchedIds := make(map[distributedTraceOwner]bool)for _, ep := range endpoints {- matchedIds[ep.Id] = true+ matchedIds[distributedTraceOwner{ep.ProjectId, ep.Id}] = true}for _, t := range tasks {- matchedIds[t.Id] = true+ matchedIds[distributedTraceOwner{t.ProjectId, t.Id}] = true}for _, a := range aiTraces {- matchedIds[a.Id] = true+ matchedIds[distributedTraceOwner{a.ProjectId, a.Id}] = true}var nodes []DistributedTraceNode+ var lookups []telemetry.SpanLookupfor _, ep := range endpoints {node := DistributedTraceNode{@@ -134,9 +136,10 @@ func (d distributedTraceController) GetDistributedTrace(c *gin.Context) {TraceType: "endpoint",Endpoint: &ep,Spans: []models.Span{},- Exception: exceptionByTraceId[ep.Id],+ Exception: exceptionByTraceId[distributedTraceOwner{ep.ProjectId, ep.Id}],}nodes = append(nodes, node)+ lookups = append(lookups, telemetry.SpanLookup{ProjectId: ep.ProjectId, TraceId: ep.Id, RecordedAt: &ep.RecordedAt})}for _, t := range tasks {@@ -146,9 +149,10 @@ func (d distributedTraceController) GetDistributedTrace(c *gin.Context) {TraceType: "task",Task: &t,Spans: []models.Span{},- Exception: exceptionByTraceId[t.Id],+ Exception: exceptionByTraceId[distributedTraceOwner{t.ProjectId, t.Id}],}nodes = append(nodes, node)+ lookups = append(lookups, telemetry.SpanLookup{ProjectId: t.ProjectId, TraceId: t.Id, RecordedAt: &t.RecordedAt})}for _, a := range aiTraces {@@ -158,13 +162,14 @@ func (d distributedTraceController) GetDistributedTrace(c *gin.Context) {TraceType: "ai_trace",AiTrace: &a,Spans: []models.Span{},- Exception: exceptionByTraceId[a.Id],+ Exception: exceptionByTraceId[distributedTraceOwner{a.ProjectId, a.Id}],}nodes = append(nodes, node)+ lookups = append(lookups, telemetry.SpanLookup{ProjectId: a.ProjectId, TraceId: a.Id, RecordedAt: &a.RecordedAt})}for _, exc := range exceptions {- if exc.TraceId != nil && matchedIds[*exc.TraceId] {+ if exc.TraceId != nil && matchedIds[distributedTraceOwner{exc.ProjectId, *exc.TraceId}] {continue}nodes = append(nodes, DistributedTraceNode{@@ -178,6 +183,16 @@ func (d distributedTraceController) GetDistributedTrace(c *gin.Context) {RecordedAt: exc.RecordedAt.Format("2006-01-02T15:04:05Z07:00"),},})+ lookup := telemetry.SpanLookup{ProjectId: exc.ProjectId, RecordedAt: &exc.RecordedAt}+ if exc.TraceId != nil {+ lookup.TraceId = *exc.TraceId+ }+ lookups = append(lookups, lookup)+ }++ if err := loadDistributedTraceSpans(ctx, nodes, lookups); err != nil {+ c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to query distributed trace spans: %w", err))+ return}if nodes == nil {@@ -190,4 +205,40 @@ func (d distributedTraceController) GetDistributedTrace(c *gin.Context) {})}+type distributedTraceOwner struct {+ projectId uuid.UUID+ traceId uuid.UUID+}++func loadDistributedTraceSpans(ctx context.Context, nodes []DistributedTraceNode, lookups []telemetry.SpanLookup) error {+ nodeIndexes := make(map[distributedTraceOwner][]int)+ queries := make([]telemetry.SpanLookup, 0, len(lookups))+ for i, lookup := range lookups {+ if lookup.TraceId == uuid.Nil {+ continue+ }+ key := distributedTraceOwner{lookup.ProjectId, lookup.TraceId}+ nodeIndexes[key] = append(nodeIndexes[key], i)+ queries = append(queries, lookup)+ }+ spans, err := telemetry.SpanRepository.FindByTraces(ctx, queries)+ if err != nil {+ return err+ }+ for _, span := range spans {+ for _, i := range nodeIndexes[distributedTraceOwner{span.ProjectId, span.TraceId}] {+ // A bulk result can cover multiple occurrences of the same ID. Keep+ // each node's window consistent with its individual detail endpoint.+ if lookups[i].RecordedAt != nil {+ from, to := shared.TraceWindowBounds(*lookups[i].RecordedAt)+ if span.RecordedAt.Before(from) || span.RecordedAt.After(to) {+ continue+ }+ }+ nodes[i].Spans = append(nodes[i].Spans, span)+ }+ }+ return nil+}+var DistributedTraceController = distributedTraceController{}diff --git a/backend/app/controllers/distributed_trace_test.go b/backend/app/controllers/distributed_trace_test.gonew file mode 100644index 00000000..1148b27e--- /dev/null+++ b/backend/app/controllers/distributed_trace_test.go@@ -0,0 +1,158 @@+//go:build !telemetry_ch && !transactional_pg && !telemetry_duckdb++package controllers++import (+ "context"+ "encoding/json"+ "net/http"+ "testing"+ "time"++ "github.com/gin-gonic/gin"+ "github.com/google/uuid"+ "github.com/tracewayapp/traceway/backend/app/db"+ "github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry"+ "github.com/tracewayapp/traceway/backend/app/repositories/transactional"+)++func TestGetDistributedTraceIncludesOwnedSpans(t *testing.T) {+ setupSetupControllerDB(t)+ tx, err := db.DB.Begin()+ if err != nil {+ t.Fatal(err)+ }+ defer tx.Rollback()+ userID, orgID := createSetupTestAccount(t, tx, "[email protected]", "owner")+ _, privateOrgID := createSetupTestAccount(t, tx, "[email protected]", "owner")+ createProject := func(name string, org int) uuid.UUID {+ project, err := transactional.ProjectRepository.CreateWithOrganization(tx, name, "opentelemetry", org)+ if err != nil {+ t.Fatal(err)+ }+ return project.Id+ }+ project1, project2 := createProject("API", orgID), createProject("Worker", orgID)+ privateProject := createProject("Private", privateOrgID)+ if err := tx.Commit(); err != nil {+ t.Fatal(err)+ }++ ctx := context.Background()+ now := time.Now().UTC().Truncate(time.Millisecond)+ later := now.Add(36 * time.Hour)+ distributedID := uuid.New()+ occurrenceID := uuid.MustParse("00000000-0000-0000-1985-a7abed0024db")+ aiID, emptyID, orphanID := uuid.New(), uuid.New(), uuid.New()+ if err := telemetry.EndpointRepository.InsertAsync(ctx, []models.Endpoint{{+ Id: occurrenceID, ProjectId: project1, Endpoint: "GET /jobs", RecordedAt: now, DistributedTraceId: &distributedID,+ }}); err != nil {+ t.Fatal(err)+ }+ if err := telemetry.TaskRepository.InsertAsync(ctx, []models.Task{+ {Id: occurrenceID, ProjectId: project2, TaskName: "worker", RecordedAt: later, DistributedTraceId: &distributedID},+ {Id: emptyID, ProjectId: project1, TaskName: "empty", RecordedAt: now, DistributedTraceId: &distributedID},+ {Id: occurrenceID, ProjectId: privateProject, TaskName: "private", RecordedAt: now, DistributedTraceId: &distributedID},+ }); err != nil {+ t.Fatal(err)+ }+ if err := telemetry.AiTraceRepository.InsertAsync(ctx, []models.AiTrace{{+ Id: aiID, ProjectId: project1, TraceName: "chat", RecordedAt: now, DistributedTraceId: &distributedID,+ }}); err != nil {+ t.Fatal(err)+ }+ if err := telemetry.ExceptionStackTraceRepository.InsertAsync(ctx, []models.ExceptionStackTrace{+ {Id: uuid.New(), ProjectId: project2, TraceId: &occurrenceID, ExceptionHash: "worker-error", RecordedAt: later, DistributedTraceId: &distributedID},+ {Id: uuid.New(), ProjectId: project2, TraceId: &orphanID, ExceptionHash: "orphan", RecordedAt: now, DistributedTraceId: &distributedID},+ {Id: uuid.New(), ProjectId: project1, ExceptionHash: "no-trace", RecordedAt: now, DistributedTraceId: &distributedID},+ }); err != nil {+ t.Fatal(err)+ }+ makeSpan := func(project, owner uuid.UUID, name string, at time.Time) models.Span {+ return models.Span{Id: uuid.New(), ProjectId: project, TraceId: owner, Name: name, StartTime: at, RecordedAt: at, Duration: time.Millisecond}+ }+ parent := uuid.New()+ first := makeSpan(project1, occurrenceID, "first", now)+ first.ParentSpanId = &parent+ first.Attributes = map[string]string{"db.system": "postgresql"}+ if err := telemetry.SpanRepository.InsertAsync(ctx, []models.Span{+ makeSpan(project1, occurrenceID, "second", now.Add(time.Second)), first,+ makeSpan(project2, occurrenceID, "worker-child", later),+ makeSpan(project1, aiID, "ai-child", now),+ makeSpan(project2, orphanID, "orphan-child", now),+ makeSpan(privateProject, occurrenceID, "private-child", now),+ makeSpan(project1, occurrenceID, "outside-window", now.Add(-25*time.Hour)),+ makeSpan(project1, distributedID, "distributed-id-is-not-owner", now),+ }); err != nil {+ t.Fatal(err)+ }++ for _, body := range []string{"{}", `{"recordedAt":"` + now.Format(time.RFC3339Nano) + `"}`} {+ c, recorder := newControllerTestContext(t, nil, userID, http.MethodPost, "/distributed-traces/"+distributedID.String(), body)+ c.Params = gin.Params{{Key: "distributedTraceId", Value: distributedID.String()}}+ DistributedTraceController.GetDistributedTrace(c)+ if recorder.Code != http.StatusOK {+ t.Fatalf("status %d: %s; errors: %v", recorder.Code, recorder.Body.String(), c.Errors)+ }+ var response DistributedTraceResponse+ if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {+ t.Fatal(err)+ }+ if len(response.Nodes) != 6 {+ t.Fatalf("got %d nodes, want 6: %s", len(response.Nodes), recorder.Body.String())+ }+ for _, node := range response.Nodes {+ if node.Spans == nil {+ t.Fatal("spans must serialize as an array")+ }+ switch {+ case node.Endpoint != nil:+ if len(node.Spans) != 2 || node.Spans[0].Name != "first" || node.Spans[1].Name != "second" {+ t.Fatalf("endpoint spans: %+v", node.Spans)+ }+ if node.Exception != nil || node.Spans[0].ParentSpanId == nil || *node.Spans[0].ParentSpanId != parent || node.Spans[0].Attributes["db.system"] != "postgresql" {+ t.Fatalf("incorrect endpoint metadata: %+v", node)+ }+ case node.Task != nil && node.Task.Id == occurrenceID:+ if node.ProjectId != project2 || len(node.Spans) != 1 || node.Spans[0].Name != "worker-child" || node.Exception == nil || node.Exception.ExceptionHash != "worker-error" {+ t.Fatalf("worker node: %+v", node)+ }+ case node.AiTrace != nil:+ if len(node.Spans) != 1 || node.Spans[0].Name != "ai-child" {+ t.Fatalf("AI spans: %+v", node.Spans)+ }+ case node.Exception != nil && node.Exception.ExceptionHash == "orphan":+ if len(node.Spans) != 1 || node.Spans[0].Name != "orphan-child" {+ t.Fatalf("orphan spans: %+v", node.Spans)+ }+ default:+ if len(node.Spans) != 0 {+ t.Fatalf("expected no spans: %+v", node)+ }+ }+ }+ }++ if _, err := db.TelemetryDB.Exec("DROP TABLE spans"); err != nil {+ t.Fatal(err)+ }+ for _, id := range []uuid.UUID{uuid.New(), distributedID} {+ c, recorder := newControllerTestContext(t, nil, userID, http.MethodPost, "/distributed-traces/"+id.String(), "{}")+ c.Params = gin.Params{{Key: "distributedTraceId", Value: id.String()}}+ DistributedTraceController.GetDistributedTrace(c)+ if id == distributedID {+ if recorder.Code != http.StatusInternalServerError || len(c.Errors) == 0 {+ t.Fatalf("span lookup failure must return 500, got %d", recorder.Code)+ }+ } else {+ var response DistributedTraceResponse+ if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {+ t.Fatal(err)+ }+ if recorder.Code != http.StatusOK || response.Nodes == nil || len(response.Nodes) != 0 {+ t.Fatalf("empty trace should skip span queries and return [], got %s", recorder.Body.String())+ }+ }+ }+}diff --git a/backend/app/repositories/telemetry/clickhouse/span.repository.go b/backend/app/repositories/telemetry/clickhouse/span.repository.goindex e7d3285c..5a752d7a 100644--- a/backend/app/repositories/telemetry/clickhouse/span.repository.go+++ b/backend/app/repositories/telemetry/clickhouse/span.repository.go@@ -5,11 +5,12 @@ package clickhouseimport ("context""encoding/json"- "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"+ "slices""time""github.com/tracewayapp/traceway/backend/app/chdb""github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared""github.com/ClickHouse/clickhouse-go/v2/lib/driver""github.com/google/uuid"@@ -49,18 +50,37 @@ func (r *spanRepository) InsertAsync(ctx context.Context, spans []models.Span) e}func (r *spanRepository) FindByTraceId(ctx context.Context, projectId, traceId uuid.UUID, recordedAt *time.Time) ([]models.Span, error) {- query := `SELECT- id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id, attributes- FROM spans- WHERE project_id = ? AND trace_id = ?`- args := []any{projectId, traceId}- if recordedAt != nil {- from, to := shared.TraceWindowBounds(*recordedAt)- query += ` AND recorded_at >= ? AND recorded_at <= ?`- args = append(args, from, to)+ query, args := shared.SpanLookupQuery([]shared.SpanLookup{{ProjectId: projectId, TraceId: traceId, RecordedAt: recordedAt}},+ func(t time.Time) any { return t.UTC() })+ return r.querySpans(ctx, query, args)+}++func (r *spanRepository) FindByTraces(ctx context.Context, lookups []shared.SpanLookup) ([]models.Span, error) {+ spans := make([]models.Span, 0)+ type spanKey struct {+ projectId, traceId, id uuid.UUID+ recordedAt time.Time+ }+ seen := make(map[spanKey]bool)+ for batch := range slices.Chunk(lookups, shared.SpanLookupBatchSize) {+ query, args := shared.SpanLookupQuery(batch, func(t time.Time) any { return t.UTC() })+ found, err := r.querySpans(ctx, query, args)+ if err != nil {+ return nil, err+ }+ for _, span := range found {+ key := spanKey{span.ProjectId, span.TraceId, span.Id, span.RecordedAt}+ if !seen[key] {+ seen[key] = true+ spans = append(spans, span)+ }+ }}- query += ` ORDER BY start_time ASC`+ slices.SortStableFunc(spans, func(a, b models.Span) int { return a.StartTime.Compare(b.StartTime) })+ return spans, nil+}+func (r *spanRepository) querySpans(ctx context.Context, query string, args []any) ([]models.Span, error) {rows, err := chdb.Conn.Query(ctx, query, args...)if err != nil {return nil, err@@ -86,7 +106,7 @@ func (r *spanRepository) FindByTraceId(ctx context.Context, projectId, traceId uspans = append(spans, s)}- return spans, nil+ return spans, rows.Err()}var SpanRepository = &spanRepository{}diff --git a/backend/app/repositories/telemetry/duckdb/span.repository.go b/backend/app/repositories/telemetry/duckdb/span.repository.goindex 7f9d5bd7..b6b82c5d 100644--- a/backend/app/repositories/telemetry/duckdb/span.repository.go+++ b/backend/app/repositories/telemetry/duckdb/span.repository.go@@ -4,8 +4,7 @@ package duckdbimport ("context"- "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"- "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/sqlitetypes"+ "slices""time""github.com/duckdb/duckdb-go/v2"@@ -13,6 +12,8 @@ import ("github.com/tracewayapp/lit/v2""github.com/tracewayapp/traceway/backend/app/db""github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/sqlitetypes")type span struct {@@ -91,28 +92,53 @@ func (r *spanRepository) InsertAsync(ctx context.Context, spans []models.Span) e}func (r *spanRepository) FindByTraceId(ctx context.Context, projectId, traceId uuid.UUID, recordedAt *time.Time) ([]models.Span, error) {- query := `SELECT id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id, attributes- FROM spans- WHERE project_id = :project_id AND trace_id = :trace_id`- params := lit.P{"project_id": projectId, "trace_id": traceId}- if recordedAt != nil {- from, to := shared.TraceWindowBounds(*recordedAt)- query += ` AND recorded_at >= :from AND recorded_at <= :to`- params["from"] = from.UTC()- params["to"] = to.UTC()+ query, args := shared.SpanLookupQuery([]shared.SpanLookup{{ProjectId: projectId, TraceId: traceId, RecordedAt: recordedAt}},+ func(t time.Time) any { return t.UTC() })+ return r.querySpans(ctx, query, args)+}++func (r *spanRepository) FindByTraces(ctx context.Context, lookups []shared.SpanLookup) ([]models.Span, error) {+ spans := make([]models.Span, 0)+ type spanKey struct {+ projectId, traceId, id uuid.UUID+ recordedAt time.Time}- query += ` ORDER BY start_time ASC`+ seen := make(map[spanKey]bool)+ for batch := range slices.Chunk(lookups, shared.SpanLookupBatchSize) {+ query, args := shared.SpanLookupQuery(batch, func(t time.Time) any { return t.UTC() })+ found, err := r.querySpans(ctx, query, args)+ if err != nil {+ return nil, err+ }+ for _, span := range found {+ key := spanKey{span.ProjectId, span.TraceId, span.Id, span.RecordedAt}+ if !seen[key] {+ seen[key] = true+ spans = append(spans, span)+ }+ }+ }+ slices.SortStableFunc(spans, func(a, b models.Span) int { return a.StartTime.Compare(b.StartTime) })+ return spans, nil+}- rows, err := lit.SelectNamed[span](db.TelemetryDB, query, params)+func (r *spanRepository) querySpans(ctx context.Context, query string, args []any) ([]models.Span, error) {+ rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)if err != nil {return nil, err}-- spans := make([]models.Span, 0, len(rows))- for _, row := range rows {+ defer rows.Close()++ spans := make([]models.Span, 0)+ for rows.Next() {+ var row span+ if err := rows.Scan(&row.Id, &row.TraceId, &row.ProjectId, &row.Name, &row.StartTime,+ &row.Duration, &row.RecordedAt, &row.ParentSpanId, &row.Attributes); err != nil {+ return nil, err+ }spans = append(spans, row.toModel())}- return spans, nil+ return spans, rows.Err()}var SpanRepository = &spanRepository{}diff --git a/backend/app/repositories/telemetry/shared/span_lookup.go b/backend/app/repositories/telemetry/shared/span_lookup.gonew file mode 100644index 00000000..7c8958d5--- /dev/null+++ b/backend/app/repositories/telemetry/shared/span_lookup.go@@ -0,0 +1,34 @@+package shared++import (+ "strings"+ "time"++ "github.com/google/uuid"+)++const SpanLookupBatchSize = 100++type SpanLookup struct {+ ProjectId uuid.UUID+ // TraceId is the owning occurrence ID, which may differ from the OTel trace ID.+ TraceId uuid.UUID+ RecordedAt *time.Time+}++func SpanLookupQuery(lookups []SpanLookup, timeValue func(time.Time) any) (string, []any) {+ conditions := make([]string, 0, len(lookups))+ args := make([]any, 0, 4*len(lookups))+ for _, lookup := range lookups {+ condition := "project_id = ? AND trace_id = ?"+ args = append(args, lookup.ProjectId, lookup.TraceId)+ if lookup.RecordedAt != nil {+ from, to := TraceWindowBounds(*lookup.RecordedAt)+ condition += " AND recorded_at >= ? AND recorded_at <= ?"+ args = append(args, timeValue(from), timeValue(to))+ }+ conditions = append(conditions, "("+condition+")")+ }+ return `SELECT id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id, attributes+ FROM spans WHERE ` + strings.Join(conditions, " OR ") + ` ORDER BY start_time ASC`, args+}diff --git a/backend/app/repositories/telemetry/span_repository_test.go b/backend/app/repositories/telemetry/span_repository_test.goindex f804ff07..4a5bbd98 100644--- a/backend/app/repositories/telemetry/span_repository_test.go+++ b/backend/app/repositories/telemetry/span_repository_test.go@@ -2,11 +2,14 @@ package telemetryimport ("context"+ "fmt"+ "reflect""testing""time""github.com/google/uuid""github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared")func TestSpanRepository_InsertAndFindByTraceId(t *testing.T) {@@ -137,3 +140,66 @@ func TestSpanRepository_ProjectIsolation(t *testing.T) {t.Errorf("expected span name 'span-p1', got %q", found[0].Name)}}++func TestSpanRepository_FindByTraces(t *testing.T) {+ setupTestDB(t)+ ctx := context.Background()+ project1, project2 := uuid.New(), uuid.New()+ now := truncateMs(time.Now().UTC())+ later := now.Add(72 * time.Hour)+ var lookups []SpanLookup+ var spans []models.Span+ for i := 0; i < shared.SpanLookupBatchSize+5; i++ {+ traceID := uuid.New()+ lookups = append(lookups, SpanLookup{ProjectId: project1, TraceId: traceID, RecordedAt: &now})+ spans = append(spans, makeSpan(project1, traceID, fmt.Sprintf("span-%d", i), now.Add(-time.Duration(i)*time.Second), time.Millisecond))+ }+ parent := uuid.New()+ spans[0].ParentSpanId = &parent+ spans[0].Attributes = map[string]string{"db.system": "postgresql"}+ // The same span ID in another project or occurrence must remain distinct.+ otherProject := spans[0]+ otherProject.ProjectId = project2+ otherProject.Name = "other-project"+ otherOccurrence := spans[0]+ otherOccurrence.StartTime, otherOccurrence.RecordedAt = later, later+ otherOccurrence.Name = "later-occurrence"+ lookups = append(lookups,+ lookups[0],+ SpanLookup{ProjectId: project2, TraceId: spans[0].TraceId, RecordedAt: &now},+ SpanLookup{ProjectId: project1, TraceId: spans[0].TraceId, RecordedAt: &later},+ )+ spans = append(spans, otherProject, otherOccurrence)+ excluded := []models.Span{+ makeSpan(uuid.New(), spans[0].TraceId, "inaccessible-project", now, time.Millisecond),+ makeSpan(project1, uuid.New(), "unrequested-owner", now, time.Millisecond),+ makeSpan(project1, spans[0].TraceId, "outside-window", now.Add(36*time.Hour), time.Millisecond),+ }+ if err := SpanRepository.InsertAsync(ctx, append(spans, excluded...)); err != nil {+ t.Fatal(err)+ }+ found, err := SpanRepository.FindByTraces(ctx, lookups)+ if err != nil {+ t.Fatal(err)+ }+ if len(found) != len(spans) {+ t.Fatalf("got %d spans, want %d", len(found), len(spans))+ }+ for i, span := range found {+ if i > 0 && span.StartTime.Before(found[i-1].StartTime) {+ t.Fatal("spans are not ordered across batches")+ }+ if span.Name == "span-0" && (span.ParentSpanId == nil || *span.ParentSpanId != parent || !reflect.DeepEqual(span.Attributes, spans[0].Attributes)) {+ t.Fatalf("span metadata did not round-trip: %+v", span)+ }+ for _, unwanted := range excluded {+ if span.Id == unwanted.Id {+ t.Fatalf("returned excluded span %s", span.Name)+ }+ }+ }+ empty, err := SpanRepository.FindByTraces(ctx, nil)+ if err != nil || len(empty) != 0 {+ t.Fatalf("empty lookup: got %v, %v", empty, err)+ }+}diff --git a/backend/app/repositories/telemetry/sqlite/span.repository.go b/backend/app/repositories/telemetry/sqlite/span.repository.goindex cab48342..b0ed3ed4 100644--- a/backend/app/repositories/telemetry/sqlite/span.repository.go+++ b/backend/app/repositories/telemetry/sqlite/span.repository.go@@ -4,14 +4,15 @@ package sqliteimport ("context"- "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"- "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/sqlitetypes"+ "slices""time""github.com/google/uuid""github.com/tracewayapp/lit/v2""github.com/tracewayapp/traceway/backend/app/db""github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/sqlitetypes")type span struct {@@ -87,28 +88,53 @@ func (r *spanRepository) InsertAsync(ctx context.Context, spans []models.Span) e}func (r *spanRepository) FindByTraceId(ctx context.Context, projectId, traceId uuid.UUID, recordedAt *time.Time) ([]models.Span, error) {- query := `SELECT id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id, attributes- FROM spans- WHERE project_id = :project_id AND trace_id = :trace_id`- params := lit.P{"project_id": projectId, "trace_id": traceId}- if recordedAt != nil {- from, to := shared.TraceWindowBounds(*recordedAt)- query += ` AND recorded_at >= :from AND recorded_at <= :to`- params["from"] = sqlitetypes.NewSQLiteTime(from)- params["to"] = sqlitetypes.NewSQLiteTime(to)+ query, args := shared.SpanLookupQuery([]shared.SpanLookup{{ProjectId: projectId, TraceId: traceId, RecordedAt: recordedAt}},+ func(t time.Time) any { return sqlitetypes.NewSQLiteTime(t) })+ return r.querySpans(ctx, query, args)+}++func (r *spanRepository) FindByTraces(ctx context.Context, lookups []shared.SpanLookup) ([]models.Span, error) {+ spans := make([]models.Span, 0)+ type spanKey struct {+ projectId, traceId, id uuid.UUID+ recordedAt time.Time+ }+ seen := make(map[spanKey]bool)+ for batch := range slices.Chunk(lookups, shared.SpanLookupBatchSize) {+ query, args := shared.SpanLookupQuery(batch, func(t time.Time) any { return sqlitetypes.NewSQLiteTime(t) })+ found, err := r.querySpans(ctx, query, args)+ if err != nil {+ return nil, err+ }+ for _, span := range found {+ key := spanKey{span.ProjectId, span.TraceId, span.Id, span.RecordedAt}+ if !seen[key] {+ seen[key] = true+ spans = append(spans, span)+ }+ }}- query += ` ORDER BY start_time ASC`+ slices.SortStableFunc(spans, func(a, b models.Span) int { return a.StartTime.Compare(b.StartTime) })+ return spans, nil+}- rows, err := lit.SelectNamed[span](db.TelemetryDB, query, params)+func (r *spanRepository) querySpans(ctx context.Context, query string, args []any) ([]models.Span, error) {+ rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)if err != nil {return nil, err}-- spans := make([]models.Span, 0, len(rows))- for _, row := range rows {+ defer rows.Close()++ spans := make([]models.Span, 0)+ for rows.Next() {+ var row span+ if err := rows.Scan(&row.Id, &row.TraceId, &row.ProjectId, &row.Name, &row.StartTime,+ &row.Duration, &row.RecordedAt, &row.ParentSpanId, &row.Attributes); err != nil {+ return nil, err+ }spans = append(spans, row.toModel())}- return spans, nil+ return spans, rows.Err()}var SpanRepository = &spanRepository{}diff --git a/backend/app/repositories/telemetry/telemetry_shared.go b/backend/app/repositories/telemetry/telemetry_shared.goindex 62186e2e..2227bbc3 100644--- a/backend/app/repositories/telemetry/telemetry_shared.go+++ b/backend/app/repositories/telemetry/telemetry_shared.go@@ -12,6 +12,7 @@ type (CheckResult = shared.CheckResultLogAttributeFilter = shared.LogAttributeFilterLogSearchParams = shared.LogSearchParams+ SpanLookup = shared.SpanLookup)var (