Skip to content

The only tool you need to know what is happening and how to fix it.

9.8 KB
Raw Blame
//go:build !telemetry_ch && !telemetry_duckdb && !transactional_pg

package otelcontrollers

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/google/uuid"
	"github.com/tracewayapp/traceway/backend/app/config"
	"github.com/tracewayapp/traceway/backend/app/db"
	"github.com/tracewayapp/traceway/backend/app/dbtest"
	"github.com/tracewayapp/traceway/backend/app/hooks"
	"github.com/tracewayapp/traceway/backend/app/middleware"
	"github.com/tracewayapp/traceway/backend/app/models"
	"github.com/tracewayapp/traceway/backend/app/repositories/telemetry"
	"github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"
	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
	commonpb "go.opentelemetry.io/proto/otlp/common/v1"
	tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
	"google.golang.org/protobuf/proto"
)

var traceTables = []string{"spans_v2", "endpoints_v2", "tasks_v2", "exceptions_v2", "ai_traces_v2"}

func traceTableCounts(t *testing.T) []int {
	t.Helper()
	counts := make([]int, len(traceTables))
	for i, table := range traceTables {
		if err := db.TelemetryDB.QueryRow("SELECT COUNT(*) FROM " + table).Scan(&counts[i]); err != nil {
			t.Fatal(err)
		}
	}
	return counts
}

func everyTableRequest() (*coltracepb.ExportTraceServiceRequest, *tracepb.Span, *tracepb.Span) {
	trace := uuid.New()
	now := uint64(time.Now().UnixNano())
	endpoint := &tracepb.Span{TraceId: trace[:], SpanId: []byte{1, 0, 0, 0, 0, 0, 0, 1}, Name: "GET /orders", Kind: tracepb.Span_SPAN_KIND_SERVER,
		StartTimeUnixNano: now, EndTimeUnixNano: now + 9000, Attributes: []*commonpb.KeyValue{strKV("http.request.method", "GET"), strKV("http.route", "/orders")}}
	child := &tracepb.Span{TraceId: trace[:], SpanId: []byte{2, 0, 0, 0, 0, 0, 0, 2}, ParentSpanId: endpoint.SpanId, Name: "query", Kind: tracepb.Span_SPAN_KIND_INTERNAL,
		StartTimeUnixNano: now + 1, EndTimeUnixNano: now + 500, Events: []*tracepb.Span_Event{{Name: "exception", TimeUnixNano: now + 2,
			Attributes: []*commonpb.KeyValue{strKV("exception.type", "PgError"), strKV("exception.message", "deadlock detected")}}}}
	task := &tracepb.Span{TraceId: trace[:], SpanId: []byte{3, 0, 0, 0, 0, 0, 0, 3}, ParentSpanId: endpoint.SpanId, Name: "process job", Kind: tracepb.Span_SPAN_KIND_CONSUMER,
		StartTimeUnixNano: now + 3, EndTimeUnixNano: now + 800}
	llm := &tracepb.Span{TraceId: trace[:], SpanId: []byte{4, 0, 0, 0, 0, 0, 0, 4}, ParentSpanId: endpoint.SpanId, Name: "chat model", Kind: tracepb.Span_SPAN_KIND_CLIENT,
		StartTimeUnixNano: now + 4, EndTimeUnixNano: now + 900, Attributes: []*commonpb.KeyValue{strKV("gen_ai.system", "anthropic"), strKV("gen_ai.request.model", "claude-sonnet-5")}}
	return spanRequest(endpoint, child, task, llm), endpoint, child
}

func exportTraces(t *testing.T, request *coltracepb.ExportTraceServiceRequest) (*httptest.ResponseRecorder, *coltracepb.ExportTraceServiceResponse) {
	t.Helper()
	payload, err := proto.Marshal(request)
	if err != nil {
		t.Fatal(err)
	}
	recorder := httptest.NewRecorder()
	c, _ := gin.CreateTestContext(recorder)
	c.Request = httptest.NewRequest(http.MethodPost, "/api/otel/v1/traces", bytes.NewReader(payload))
	c.Request.Header.Set("Content-Type", "application/x-protobuf")
	c.Set(middleware.ProjectIdContextKey, testProjectId)
	OtelController.ExportTraces(c)
	c.Writer.WriteHeaderNow()
	var response coltracepb.ExportTraceServiceResponse
	if recorder.Code == http.StatusOK {
		if err := proto.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
			t.Fatal(err)
		}
	}
	return recorder, &response
}

func setupTraceWrites(t *testing.T) {
	t.Helper()
	dbtest.SetupSQLite(t)
	previousConfig, previousStore := config.Config, traceStore
	config.Config = &config.Cfg{}
	setFakeStore(t, nil)
	gin.SetMode(gin.TestMode)
	t.Cleanup(func() { config.Config, traceStore = previousConfig, previousStore })
}

func TestExportTracesFailureAtEachWriteStage(t *testing.T) {
	stages := []struct {
		name      string
		inject    func(error)
		persisted []int
	}{
		{"spans", func(err error) {
			traceStore.spans = func(context.Context, []models.OtelSpan) ([]shared.SpanOwner, error) { return nil, err }
		}, []int{0, 0, 0, 0, 0}},
		{"endpoints", func(err error) { traceStore.endpoints = func(context.Context, []models.Endpoint) error { return err } }, []int{4, 0, 0, 0, 0}},
		{"tasks", func(err error) { traceStore.tasks = func(context.Context, []models.Task) error { return err } }, []int{4, 1, 0, 0, 0}},
		{"exceptions", func(err error) {
			traceStore.exceptions = func(context.Context, []models.ExceptionStackTrace) error { return err }
		}, []int{4, 1, 1, 0, 0}},
		{"ai traces", func(err error) { traceStore.aiTraces = func(context.Context, []models.AiTrace) error { return err } }, []int{4, 1, 1, 1, 0}},
	}
	failures := []struct {
		err        error
		status     int
		retryAfter string
	}{
		{errors.New("no such column: resource_pb"), http.StatusInternalServerError, ""},
		{fmt.Errorf("insert timed out: %w", context.DeadlineExceeded), http.StatusServiceUnavailable, "2"},
	}
	for _, stage := range stages {
		for _, failure := range failures {
			t.Run(fmt.Sprintf("%s/%d", stage.name, failure.status), func(t *testing.T) {
				setupTraceWrites(t)
				stage.inject(failure.err)
				request, _, _ := everyTableRequest()
				recorder, _ := exportTraces(t, request)
				if recorder.Code != failure.status || recorder.Header().Get("Retry-After") != failure.retryAfter {
					t.Fatalf("status %d Retry-After %q, want %d %q", recorder.Code, recorder.Header().Get("Retry-After"), failure.status, failure.retryAfter)
				}
				if got := traceTableCounts(t); fmt.Sprint(got) != fmt.Sprint(stage.persisted) {
					t.Fatalf("persisted rows %v, want %v in %v", got, stage.persisted, traceTables)
				}
			})
		}
	}
}

func TestExportTracesRetryDuplicatesAnalyticsButNotTheGraph(t *testing.T) {
	setupTraceWrites(t)
	request, endpoint, _ := everyTableRequest()
	for range 2 {
		if recorder, _ := exportTraces(t, request); recorder.Code != http.StatusOK {
			t.Fatalf("export failed: %d", recorder.Code)
		}
	}
	// Writes are separate and append-only, so a retried export is stored twice. Only graph reads deduplicate.
	if got := traceTableCounts(t); fmt.Sprint(got) != fmt.Sprint([]int{8, 2, 2, 2, 2}) {
		t.Fatalf("a retried export is at-least-once: got %v in %v", got, traceTables)
	}
	id := otelOccurrenceID(testProjectId, endpoint)
	stored, err := telemetry.EndpointRepository.FindById(context.Background(), testProjectId, id, nil)
	if err != nil || stored == nil {
		t.Fatalf("missing endpoint: %v", err)
	}
	graph, err := telemetry.SpanRepository.FindGraph(context.Background(), shared.NewSpanLookup(testProjectId, stored.TraceId, stored.SpanId, stored.RecordedAt))
	if err != nil {
		t.Fatal(err)
	}
	if len(graph.Spans) != 3 {
		t.Fatalf("graph reads must deduplicate the retry: %d spans", len(graph.Spans))
	}
}

func TestExportTracesReportsRowsStorageRejected(t *testing.T) {
	setupTraceWrites(t)
	store := traceStore.spans
	traceStore.spans = func(ctx context.Context, spans []models.OtelSpan) ([]shared.SpanOwner, error) {
		stored, err := store(ctx, spans[:len(spans)-2])
		for _, span := range spans[len(spans)-2:] {
			stored = append(stored, shared.SpanOwner{ProjectId: span.ProjectId, TraceId: span.TraceId, SpanId: span.SpanId})
		}
		return stored, err
	}
	request, _, _ := everyTableRequest()
	request.ResourceSpans[0].ScopeSpans[0].Spans = append(request.ResourceSpans[0].ScopeSpans[0].Spans, &tracepb.Span{TraceId: []byte{1}, SpanId: []byte{1}})
	recorder, response := exportTraces(t, request)
	partial := response.GetPartialSuccess()
	if recorder.Code != http.StatusOK || partial.GetRejectedSpans() != 3 {
		t.Fatalf("status %d, partial success %v", recorder.Code, partial)
	}
	if got := traceTableCounts(t); fmt.Sprint(got) != fmt.Sprint([]int{2, 1, 0, 1, 0}) {
		t.Fatalf("rejected task/AI spans must not produce projections: %v", got)
	}

	if !strings.Contains(partial.GetErrorMessage(), "1 spans need valid non-zero trace and span IDs") || !strings.Contains(partial.GetErrorMessage(), "2 spans could not be stored") {
		t.Fatalf("the exporter must learn why spans were rejected: %q", partial.GetErrorMessage())
	}
}

func TestExportTracesCountsInvalidIDsWhenQuotaSuppressesWrites(t *testing.T) {
	for _, tt := range []struct {
		name       string
		permission hooks.IngestPermission
		counts     []int
	}{
		{"exceptions only", hooks.IngestPermission{Exceptions: true}, []int{0, 0, 0, 1, 0}},
		{"data only", hooks.IngestPermission{Data: true}, []int{4, 1, 1, 0, 1}},
		{"neither", hooks.IngestPermission{}, []int{0, 0, 0, 0, 0}},
	} {
		t.Run(tt.name, func(t *testing.T) {
			setupTraceWrites(t)
			hooks.RegisterIngestPermissionHook(func(int) hooks.IngestPermission { return tt.permission })
			t.Cleanup(func() { hooks.RegisterIngestPermissionHook(nil) })
			request, _, _ := everyTableRequest()
			request.ResourceSpans[0].ScopeSpans[0].Spans = append(request.ResourceSpans[0].ScopeSpans[0].Spans, &tracepb.Span{})
			payload, err := proto.Marshal(request)
			if err != nil {
				t.Fatal(err)
			}
			recorder := httptest.NewRecorder()
			c, _ := gin.CreateTestContext(recorder)
			c.Request = httptest.NewRequest(http.MethodPost, "/api/otel/v1/traces", bytes.NewReader(payload))
			c.Request.Header.Set("Content-Type", "application/x-protobuf")
			c.Set(middleware.ProjectIdContextKey, testProjectId)
			orgID := 1
			c.Set(middleware.ProjectContextKey, &models.Project{Id: testProjectId, OrganizationId: &orgID})
			OtelController.ExportTraces(c)
			var response coltracepb.ExportTraceServiceResponse
			if recorder.Code != http.StatusOK || proto.Unmarshal(recorder.Body.Bytes(), &response) != nil || response.GetPartialSuccess().GetRejectedSpans() != 1 {
				t.Fatalf("status %d, response %v", recorder.Code, &response)
			}
			if got := traceTableCounts(t); fmt.Sprint(got) != fmt.Sprint(tt.counts) {
				t.Fatalf("persisted rows %v, want %v", got, tt.counts)
			}
		})
	}
}